net/ipv6/route.c

   1 /*
   2  *      Linux INET6 implementation
   3  *      FIB front-end.
   4  *
   5  *      Authors:
   6  *      Pedro Roque             <roque@di.fc.ul.pt>
   7  *
   8  *      This program is free software; you can redistribute it and/or
   9  *      modify it under the terms of the GNU General Public License
  10  *      as published by the Free Software Foundation; either version
  11  *      2 of the License, or (at your option) any later version.
  12  */
  13
  14 /*      Changes:
  15  *
  16  *      YOSHIFUJI Hideaki @USAGI
  17  *              reworked default router selection.
  18  *              - respect outgoing interface
  19  *              - select from (probably) reachable routers (i.e.
  20  *              routers in REACHABLE, STALE, DELAY or PROBE states).
  21  *              - always select the same router if it is (probably)
  22  *              reachable.  otherwise, round-robin the list.
  23  *      Ville Nuorvala
  24  *              Fixed routing subtrees.
  25  */
  26
  27 #define pr_fmt(fmt) "IPv6: " fmt
  28
  29 #include <linux/capability.h>
  30 #include <linux/errno.h>
  31 #include <linux/export.h>
  32 #include <linux/types.h>
  33 #include <linux/times.h>
  34 #include <linux/socket.h>
  35 #include <linux/sockios.h>
  36 #include <linux/net.h>
  37 #include <linux/route.h>
  38 #include <linux/netdevice.h>
  39 #include <linux/in6.h>
  40 #include <linux/mroute6.h>
  41 #include <linux/init.h>
  42 #include <linux/if_arp.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/seq_file.h>
  45 #include <linux/nsproxy.h>
  46 #include <linux/slab.h>
  47 #include <linux/jhash.h>
  48 #include <net/net_namespace.h>
  49 #include <net/snmp.h>
  50 #include <net/ipv6.h>
  51 #include <net/ip6_fib.h>
  52 #include <net/ip6_route.h>
  53 #include <net/ndisc.h>
  54 #include <net/addrconf.h>
  55 #include <net/tcp.h>
  56 #include <linux/rtnetlink.h>
  57 #include <net/dst.h>
  58 #include <net/dst_metadata.h>
  59 #include <net/xfrm.h>
  60 #include <net/netevent.h>
  61 #include <net/netlink.h>
  62 #include <net/nexthop.h>
  63 #include <net/lwtunnel.h>
  64 #include <net/ip_tunnels.h>
  65 #include <net/l3mdev.h>
  66 #include <net/ip.h>
  67 #include <linux/uaccess.h>
  68
  69 #ifdef CONFIG_SYSCTL
  70 #include <linux/sysctl.h>
  71 #endif
  72
  73 static int ip6_rt_type_to_error(u8 fib6_type);
  74
  75 #define CREATE_TRACE_POINTS
  76 #include <trace/events/fib6.h>
  77 EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup);
  78 #undef CREATE_TRACE_POINTS
  79
  80 enum rt6_nud_state {
  81         RT6_NUD_FAIL_HARD = -3,
  82         RT6_NUD_FAIL_PROBE = -2,
  83         RT6_NUD_FAIL_DO_RR = -1,
  84         RT6_NUD_SUCCEED = 1
  85 };
  86
  87 static struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie);
  88 static unsigned int      ip6_default_advmss(const struct dst_entry *dst);
  89 static unsigned int      ip6_mtu(const struct dst_entry *dst);
  90 static struct dst_entry *ip6_negative_advice(struct dst_entry *);
  91 static void             ip6_dst_destroy(struct dst_entry *);
  92 static void             ip6_dst_ifdown(struct dst_entry *,
  93                                        struct net_device *dev, int how);
  94 static int               ip6_dst_gc(struct dst_ops *ops);
  95
  96 static int              ip6_pkt_discard(struct sk_buff *skb);
  97 static int              ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
  98 static int              ip6_pkt_prohibit(struct sk_buff *skb);
  99 static int              ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb);
 100 static void             ip6_link_failure(struct sk_buff *skb);
 101 static void             ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 102                                            struct sk_buff *skb, u32 mtu);
 103 static void             rt6_do_redirect(struct dst_entry *dst, struct sock *sk,
 104                                         struct sk_buff *skb);
 105 static int rt6_score_route(struct fib6_info *rt, int oif, int strict);
 106 static size_t rt6_nlmsg_size(struct fib6_info *rt);
 107 static int rt6_fill_node(struct net *net, struct sk_buff *skb,
 108                          struct fib6_info *rt, struct dst_entry *dst,
 109                          struct in6_addr *dest, struct in6_addr *src,
 110                          int iif, int type, u32 portid, u32 seq,
 111                          unsigned int flags);
 112 static struct rt6_info *rt6_find_cached_rt(struct fib6_info *rt,
 113                                            struct in6_addr *daddr,
 114                                            struct in6_addr *saddr);
 115
 116 #ifdef CONFIG_IPV6_ROUTE_INFO
 117 static struct fib6_info *rt6_add_route_info(struct net *net,
 118                                            const struct in6_addr *prefix, int prefixlen,
 119                                            const struct in6_addr *gwaddr,
 120                                            struct net_device *dev,
 121                                            unsigned int pref);
 122 static struct fib6_info *rt6_get_route_info(struct net *net,
 123                                            const struct in6_addr *prefix, int prefixlen,
 124                                            const struct in6_addr *gwaddr,
 125                                            struct net_device *dev);
 126 #endif
 127
 128 struct uncached_list {
 129         spinlock_t              lock;
 130         struct list_head        head;
 131 };
 132
 133 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list);
 134
 135 void rt6_uncached_list_add(struct rt6_info *rt)
 136 {
 137         struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list);
 138
 139         rt->rt6i_uncached_list = ul;
 140
 141         spin_lock_bh(&ul->lock);
 142         list_add_tail(&rt->rt6i_uncached, &ul->head);
 143         spin_unlock_bh(&ul->lock);
 144 }
 145
 146 void rt6_uncached_list_del(struct rt6_info *rt)
 147 {
 148         if (!list_empty(&rt->rt6i_uncached)) {
 149                 struct uncached_list *ul = rt->rt6i_uncached_list;
 150                 struct net *net = dev_net(rt->dst.dev);
 151
 152                 spin_lock_bh(&ul->lock);
 153                 list_del(&rt->rt6i_uncached);
 154                 atomic_dec(&net->ipv6.rt6_stats->fib_rt_uncache);
 155                 spin_unlock_bh(&ul->lock);
 156         }
 157 }
 158
 159 static void rt6_uncached_list_flush_dev(struct net *net, struct net_device *dev)
 160 {
 161         struct net_device *loopback_dev = net->loopback_dev;
 162         int cpu;
 163
 164         if (dev == loopback_dev)
 165                 return;
 166
 167         for_each_possible_cpu(cpu) {
 168                 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
 169                 struct rt6_info *rt;
 170
 171                 spin_lock_bh(&ul->lock);
 172                 list_for_each_entry(rt, &ul->head, rt6i_uncached) {
 173                         struct inet6_dev *rt_idev = rt->rt6i_idev;
 174                         struct net_device *rt_dev = rt->dst.dev;
 175
 176                         if (rt_idev->dev == dev) {
 177                                 rt->rt6i_idev = in6_dev_get(loopback_dev);
 178                                 in6_dev_put(rt_idev);
 179                         }
 180
 181                         if (rt_dev == dev) {
 182                                 rt->dst.dev = loopback_dev;
 183                                 dev_hold(rt->dst.dev);
 184                                 dev_put(rt_dev);
 185                         }
 186                 }
 187                 spin_unlock_bh(&ul->lock);
 188         }
 189 }
 190
 191 static inline const void *choose_neigh_daddr(const struct in6_addr *p,
 192                                              struct sk_buff *skb,
 193                                              const void *daddr)
 194 {
 195         if (!ipv6_addr_any(p))
 196                 return (const void *) p;
 197         else if (skb)
 198                 return &ipv6_hdr(skb)->daddr;
 199         return daddr;
 200 }
 201
 202 struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw,
 203                                    struct net_device *dev,
 204                                    struct sk_buff *skb,
 205                                    const void *daddr)
 206 {
 207         struct neighbour *n;
 208
 209         daddr = choose_neigh_daddr(gw, skb, daddr);
 210         n = __ipv6_neigh_lookup(dev, daddr);
 211         if (n)
 212                 return n;
 213         return neigh_create(&nd_tbl, daddr, dev);
 214 }
 215
 216 static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst,
 217                                               struct sk_buff *skb,
 218                                               const void *daddr)
 219 {
 220         const struct rt6_info *rt = container_of(dst, struct rt6_info, dst);
 221
 222         return ip6_neigh_lookup(&rt->rt6i_gateway, dst->dev, skb, daddr);
 223 }
 224
 225 static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr)
 226 {
 227         struct net_device *dev = dst->dev;
 228         struct rt6_info *rt = (struct rt6_info *)dst;
 229
 230         daddr = choose_neigh_daddr(&rt->rt6i_gateway, NULL, daddr);
 231         if (!daddr)
 232                 return;
 233         if (dev->flags & (IFF_NOARP | IFF_LOOPBACK))
 234                 return;
 235         if (ipv6_addr_is_multicast((const struct in6_addr *)daddr))
 236                 return;
 237         __ipv6_confirm_neigh(dev, daddr);
 238 }
 239
 240 static struct dst_ops ip6_dst_ops_template = {
 241         .family                 =       AF_INET6,
 242         .gc                     =       ip6_dst_gc,
 243         .gc_thresh              =       1024,
 244         .check                  =       ip6_dst_check,
 245         .default_advmss         =       ip6_default_advmss,
 246         .mtu                    =       ip6_mtu,
 247         .cow_metrics            =       dst_cow_metrics_generic,
 248         .destroy                =       ip6_dst_destroy,
 249         .ifdown                 =       ip6_dst_ifdown,
 250         .negative_advice        =       ip6_negative_advice,
 251         .link_failure           =       ip6_link_failure,
 252         .update_pmtu            =       ip6_rt_update_pmtu,
 253         .redirect               =       rt6_do_redirect,
 254         .local_out              =       __ip6_local_out,
 255         .neigh_lookup           =       ip6_dst_neigh_lookup,
 256         .confirm_neigh          =       ip6_confirm_neigh,
 257 };
 258
 259 static unsigned int ip6_blackhole_mtu(const struct dst_entry *dst)
 260 {
 261         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
 262
 263         return mtu ? : dst->dev->mtu;
 264 }
 265
 266 static void ip6_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
 267                                          struct sk_buff *skb, u32 mtu)
 268 {
 269 }
 270
 271 static void ip6_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
 272                                       struct sk_buff *skb)
 273 {
 274 }
 275
 276 static struct dst_ops ip6_dst_blackhole_ops = {
 277         .family                 =       AF_INET6,
 278         .destroy                =       ip6_dst_destroy,
 279         .check                  =       ip6_dst_check,
 280         .mtu                    =       ip6_blackhole_mtu,
 281         .default_advmss         =       ip6_default_advmss,
 282         .update_pmtu            =       ip6_rt_blackhole_update_pmtu,
 283         .redirect               =       ip6_rt_blackhole_redirect,
 284         .cow_metrics            =       dst_cow_metrics_generic,
 285         .neigh_lookup           =       ip6_dst_neigh_lookup,
 286 };
 287
 288 static const u32 ip6_template_metrics[RTAX_MAX] = {
 289         [RTAX_HOPLIMIT - 1] = 0,
 290 };
 291
 292 static const struct fib6_info fib6_null_entry_template = {
 293         .fib6_flags     = (RTF_REJECT | RTF_NONEXTHOP),
 294         .fib6_protocol  = RTPROT_KERNEL,
 295         .fib6_metric    = ~(u32)0,
 296         .fib6_ref       = ATOMIC_INIT(1),
 297         .fib6_type      = RTN_UNREACHABLE,
 298         .fib6_metrics   = (struct dst_metrics *)&dst_default_metrics,
 299 };
 300
 301 static const struct rt6_info ip6_null_entry_template = {
 302         .dst = {
 303                 .__refcnt       = ATOMIC_INIT(1),
 304                 .__use          = 1,
 305                 .obsolete       = DST_OBSOLETE_FORCE_CHK,
 306                 .error          = -ENETUNREACH,
 307                 .input          = ip6_pkt_discard,
 308                 .output         = ip6_pkt_discard_out,
 309         },
 310         .rt6i_flags     = (RTF_REJECT | RTF_NONEXTHOP),
 311 };
 312
 313 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
 314
 315 static const struct rt6_info ip6_prohibit_entry_template = {
 316         .dst = {
 317                 .__refcnt       = ATOMIC_INIT(1),
 318                 .__use          = 1,
 319                 .obsolete       = DST_OBSOLETE_FORCE_CHK,
 320                 .error          = -EACCES,
 321                 .input          = ip6_pkt_prohibit,
 322                 .output         = ip6_pkt_prohibit_out,
 323         },
 324         .rt6i_flags     = (RTF_REJECT | RTF_NONEXTHOP),
 325 };
 326
 327 static const struct rt6_info ip6_blk_hole_entry_template = {
 328         .dst = {
 329                 .__refcnt       = ATOMIC_INIT(1),
 330                 .__use          = 1,
 331                 .obsolete       = DST_OBSOLETE_FORCE_CHK,
 332                 .error          = -EINVAL,
 333                 .input          = dst_discard,
 334                 .output         = dst_discard_out,
 335         },
 336         .rt6i_flags     = (RTF_REJECT | RTF_NONEXTHOP),
 337 };
 338
 339 #endif
 340
 341 static void rt6_info_init(struct rt6_info *rt)
 342 {
 343         struct dst_entry *dst = &rt->dst;
 344
 345         memset(dst + 1, 0, sizeof(*rt) - sizeof(*dst));
 346         INIT_LIST_HEAD(&rt->rt6i_uncached);
 347 }
 348
 349 /* allocate dst with ip6_dst_ops */
 350 struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev,
 351                                int flags)
 352 {
 353         struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev,
 354                                         1, DST_OBSOLETE_FORCE_CHK, flags);
 355
 356         if (rt) {
 357                 rt6_info_init(rt);
 358                 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc);
 359         }
 360
 361         return rt;
 362 }
 363 EXPORT_SYMBOL(ip6_dst_alloc);
 364
 365 static void ip6_dst_destroy(struct dst_entry *dst)
 366 {
 367         struct rt6_info *rt = (struct rt6_info *)dst;
 368         struct fib6_info *from;
 369         struct inet6_dev *idev;
 370
 371         ip_dst_metrics_put(dst);
 372         rt6_uncached_list_del(rt);
 373
 374         idev = rt->rt6i_idev;
 375         if (idev) {
 376                 rt->rt6i_idev = NULL;
 377                 in6_dev_put(idev);
 378         }
 379
 380         rcu_read_lock();
 381         from = rcu_dereference(rt->from);
 382         rcu_assign_pointer(rt->from, NULL);
 383         fib6_info_release(from);
 384         rcu_read_unlock();
 385 }
 386
 387 static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
 388                            int how)
 389 {
 390         struct rt6_info *rt = (struct rt6_info *)dst;
 391         struct inet6_dev *idev = rt->rt6i_idev;
 392         struct net_device *loopback_dev =
 393                 dev_net(dev)->loopback_dev;
 394
 395         if (idev && idev->dev != loopback_dev) {
 396                 struct inet6_dev *loopback_idev = in6_dev_get(loopback_dev);
 397                 if (loopback_idev) {
 398                         rt->rt6i_idev = loopback_idev;
 399                         in6_dev_put(idev);
 400                 }
 401         }
 402 }
 403
 404 static bool __rt6_check_expired(const struct rt6_info *rt)
 405 {
 406         if (rt->rt6i_flags & RTF_EXPIRES)
 407                 return time_after(jiffies, rt->dst.expires);
 408         else
 409                 return false;
 410 }
 411
 412 static bool rt6_check_expired(const struct rt6_info *rt)
 413 {
 414         struct fib6_info *from;
 415
 416         from = rcu_dereference(rt->from);
 417
 418         if (rt->rt6i_flags & RTF_EXPIRES) {
 419                 if (time_after(jiffies, rt->dst.expires))
 420                         return true;
 421         } else if (from) {
 422                 return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK ||
 423                         fib6_check_expired(from);
 424         }
 425         return false;
 426 }
 427
 428 struct fib6_info *fib6_multipath_select(const struct net *net,
 429                                         struct fib6_info *match,
 430                                         struct flowi6 *fl6, int oif,
 431                                         const struct sk_buff *skb,
 432                                         int strict)
 433 {
 434         struct fib6_info *sibling, *next_sibling;
 435
 436         /* We might have already computed the hash for ICMPv6 errors. In such
 437          * case it will always be non-zero. Otherwise now is the time to do it.
 438          */
 439         if (!fl6->mp_hash)
 440                 fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL);
 441
 442         if (fl6->mp_hash <= atomic_read(&match->fib6_nh.nh_upper_bound))
 443                 return match;
 444
 445         list_for_each_entry_safe(sibling, next_sibling, &match->fib6_siblings,
 446                                  fib6_siblings) {
 447                 int nh_upper_bound;
 448
 449                 nh_upper_bound = atomic_read(&sibling->fib6_nh.nh_upper_bound);
 450                 if (fl6->mp_hash > nh_upper_bound)
 451                         continue;
 452                 if (rt6_score_route(sibling, oif, strict) < 0)
 453                         break;
 454                 match = sibling;
 455                 break;
 456         }
 457
 458         return match;
 459 }
 460
 461 /*
 462  *      Route lookup. rcu_read_lock() should be held.
 463  */
 464
 465 static inline struct fib6_info *rt6_device_match(struct net *net,
 466                                                  struct fib6_info *rt,
 467                                                     const struct in6_addr *saddr,
 468                                                     int oif,
 469                                                     int flags)
 470 {
 471         struct fib6_info *sprt;
 472
 473         if (!oif && ipv6_addr_any(saddr) &&
 474             !(rt->fib6_nh.nh_flags & RTNH_F_DEAD))
 475                 return rt;
 476
 477         for (sprt = rt; sprt; sprt = rcu_dereference(sprt->fib6_next)) {
 478                 const struct net_device *dev = sprt->fib6_nh.nh_dev;
 479
 480                 if (sprt->fib6_nh.nh_flags & RTNH_F_DEAD)
 481                         continue;
 482
 483                 if (oif) {
 484                         if (dev->ifindex == oif)
 485                                 return sprt;
 486                 } else {
 487                         if (ipv6_chk_addr(net, saddr, dev,
 488                                           flags & RT6_LOOKUP_F_IFACE))
 489                                 return sprt;
 490                 }
 491         }
 492
 493         if (oif && flags & RT6_LOOKUP_F_IFACE)
 494                 return net->ipv6.fib6_null_entry;
 495
 496         return rt->fib6_nh.nh_flags & RTNH_F_DEAD ? net->ipv6.fib6_null_entry : rt;
 497 }
 498
 499 #ifdef CONFIG_IPV6_ROUTER_PREF
 500 struct __rt6_probe_work {
 501         struct work_struct work;
 502         struct in6_addr target;
 503         struct net_device *dev;
 504 };
 505
 506 static void rt6_probe_deferred(struct work_struct *w)
 507 {
 508         struct in6_addr mcaddr;
 509         struct __rt6_probe_work *work =
 510                 container_of(w, struct __rt6_probe_work, work);
 511
 512         addrconf_addr_solict_mult(&work->target, &mcaddr);
 513         ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0);
 514         dev_put(work->dev);
 515         kfree(work);
 516 }
 517
 518 static void rt6_probe(struct fib6_info *rt)
 519 {
 520         struct __rt6_probe_work *work;
 521         const struct in6_addr *nh_gw;
 522         struct neighbour *neigh;
 523         struct net_device *dev;
 524
 525         /*
 526          * Okay, this does not seem to be appropriate
 527          * for now, however, we need to check if it
 528          * is really so; aka Router Reachability Probing.
 529          *
 530          * Router Reachability Probe MUST be rate-limited
 531          * to no more than one per minute.
 532          */
 533         if (!rt || !(rt->fib6_flags & RTF_GATEWAY))
 534                 return;
 535
 536         nh_gw = &rt->fib6_nh.nh_gw;
 537         dev = rt->fib6_nh.nh_dev;
 538         rcu_read_lock_bh();
 539         neigh = __ipv6_neigh_lookup_noref(dev, nh_gw);
 540         if (neigh) {
 541                 struct inet6_dev *idev;
 542
 543                 if (neigh->nud_state & NUD_VALID)
 544                         goto out;
 545
 546                 idev = __in6_dev_get(dev);
 547                 work = NULL;
 548                 write_lock(&neigh->lock);
 549                 if (!(neigh->nud_state & NUD_VALID) &&
 550                     time_after(jiffies,
 551                                neigh->updated + idev->cnf.rtr_probe_interval)) {
 552                         work = kmalloc(sizeof(*work), GFP_ATOMIC);
 553                         if (work)
 554                                 __neigh_set_probe_once(neigh);
 555                 }
 556                 write_unlock(&neigh->lock);
 557         } else {
 558                 work = kmalloc(sizeof(*work), GFP_ATOMIC);
 559         }
 560
 561         if (work) {
 562                 INIT_WORK(&work->work, rt6_probe_deferred);
 563                 work->target = *nh_gw;
 564                 dev_hold(dev);
 565                 work->dev = dev;
 566                 schedule_work(&work->work);
 567         }
 568
 569 out:
 570         rcu_read_unlock_bh();
 571 }
 572 #else
 573 static inline void rt6_probe(struct fib6_info *rt)
 574 {
 575 }
 576 #endif
 577
 578 /*
 579  * Default Router Selection (RFC 2461 6.3.6)
 580  */
 581 static inline int rt6_check_dev(struct fib6_info *rt, int oif)
 582 {
 583         const struct net_device *dev = rt->fib6_nh.nh_dev;
 584
 585         if (!oif || dev->ifindex == oif)
 586                 return 2;
 587         return 0;
 588 }
 589
 590 static inline enum rt6_nud_state rt6_check_neigh(struct fib6_info *rt)
 591 {
 592         enum rt6_nud_state ret = RT6_NUD_FAIL_HARD;
 593         struct neighbour *neigh;
 594
 595         if (rt->fib6_flags & RTF_NONEXTHOP ||
 596             !(rt->fib6_flags & RTF_GATEWAY))
 597                 return RT6_NUD_SUCCEED;
 598
 599         rcu_read_lock_bh();
 600         neigh = __ipv6_neigh_lookup_noref(rt->fib6_nh.nh_dev,
 601                                           &rt->fib6_nh.nh_gw);
 602         if (neigh) {
 603                 read_lock(&neigh->lock);
 604                 if (neigh->nud_state & NUD_VALID)
 605                         ret = RT6_NUD_SUCCEED;
 606 #ifdef CONFIG_IPV6_ROUTER_PREF
 607                 else if (!(neigh->nud_state & NUD_FAILED))
 608                         ret = RT6_NUD_SUCCEED;
 609                 else
 610                         ret = RT6_NUD_FAIL_PROBE;
 611 #endif
 612                 read_unlock(&neigh->lock);
 613         } else {
 614                 ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ?
 615                       RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR;
 616         }
 617         rcu_read_unlock_bh();
 618
 619         return ret;
 620 }
 621
 622 static int rt6_score_route(struct fib6_info *rt, int oif, int strict)
 623 {
 624         int m;
 625
 626         m = rt6_check_dev(rt, oif);
 627         if (!m && (strict & RT6_LOOKUP_F_IFACE))
 628                 return RT6_NUD_FAIL_HARD;
 629 #ifdef CONFIG_IPV6_ROUTER_PREF
 630         m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(rt->fib6_flags)) << 2;
 631 #endif
 632         if (strict & RT6_LOOKUP_F_REACHABLE) {
 633                 int n = rt6_check_neigh(rt);
 634                 if (n < 0)
 635                         return n;
 636         }
 637         return m;
 638 }
 639
 640 /* called with rc_read_lock held */
 641 static inline bool fib6_ignore_linkdown(const struct fib6_info *f6i)
 642 {
 643         const struct net_device *dev = fib6_info_nh_dev(f6i);
 644         bool rc = false;
 645
 646         if (dev) {
 647                 const struct inet6_dev *idev = __in6_dev_get(dev);
 648
 649                 rc = !!idev->cnf.ignore_routes_with_linkdown;
 650         }
 651
 652         return rc;
 653 }
 654
 655 static struct fib6_info *find_match(struct fib6_info *rt, int oif, int strict,
 656                                    int *mpri, struct fib6_info *match,
 657                                    bool *do_rr)
 658 {
 659         int m;
 660         bool match_do_rr = false;
 661
 662         if (rt->fib6_nh.nh_flags & RTNH_F_DEAD)
 663                 goto out;
 664
 665         if (fib6_ignore_linkdown(rt) &&
 666             rt->fib6_nh.nh_flags & RTNH_F_LINKDOWN &&
 667             !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE))
 668                 goto out;
 669
 670         if (fib6_check_expired(rt))
 671                 goto out;
 672
 673         m = rt6_score_route(rt, oif, strict);
 674         if (m == RT6_NUD_FAIL_DO_RR) {
 675                 match_do_rr = true;
 676                 m = 0; /* lowest valid score */
 677         } else if (m == RT6_NUD_FAIL_HARD) {
 678                 goto out;
 679         }
 680
 681         if (strict & RT6_LOOKUP_F_REACHABLE)
 682                 rt6_probe(rt);
 683
 684         /* note that m can be RT6_NUD_FAIL_PROBE at this point */
 685         if (m > *mpri) {
 686                 *do_rr = match_do_rr;
 687                 *mpri = m;
 688                 match = rt;
 689         }
 690 out:
 691         return match;
 692 }
 693
 694 static struct fib6_info *find_rr_leaf(struct fib6_node *fn,
 695                                      struct fib6_info *leaf,
 696                                      struct fib6_info *rr_head,
 697                                      u32 metric, int oif, int strict,
 698                                      bool *do_rr)
 699 {
 700         struct fib6_info *rt, *match, *cont;
 701         int mpri = -1;
 702
 703         match = NULL;
 704         cont = NULL;
 705         for (rt = rr_head; rt; rt = rcu_dereference(rt->fib6_next)) {
 706                 if (rt->fib6_metric != metric) {
 707                         cont = rt;
 708                         break;
 709                 }
 710
 711                 match = find_match(rt, oif, strict, &mpri, match, do_rr);
 712         }
 713
 714         for (rt = leaf; rt && rt != rr_head;
 715              rt = rcu_dereference(rt->fib6_next)) {
 716                 if (rt->fib6_metric != metric) {
 717                         cont = rt;
 718                         break;
 719                 }
 720
 721                 match = find_match(rt, oif, strict, &mpri, match, do_rr);
 722         }
 723
 724         if (match || !cont)
 725                 return match;
 726
 727         for (rt = cont; rt; rt = rcu_dereference(rt->fib6_next))
 728                 match = find_match(rt, oif, strict, &mpri, match, do_rr);
 729
 730         return match;
 731 }
 732
 733 static struct fib6_info *rt6_select(struct net *net, struct fib6_node *fn,
 734                                    int oif, int strict)
 735 {
 736         struct fib6_info *leaf = rcu_dereference(fn->leaf);
 737         struct fib6_info *match, *rt0;
 738         bool do_rr = false;
 739         int key_plen;
 740
 741         if (!leaf || leaf == net->ipv6.fib6_null_entry)
 742                 return net->ipv6.fib6_null_entry;
 743
 744         rt0 = rcu_dereference(fn->rr_ptr);
 745         if (!rt0)
 746                 rt0 = leaf;
 747
 748         /* Double check to make sure fn is not an intermediate node
 749          * and fn->leaf does not points to its child's leaf
 750          * (This might happen if all routes under fn are deleted from
 751          * the tree and fib6_repair_tree() is called on the node.)
 752          */
 753         key_plen = rt0->fib6_dst.plen;
 754 #ifdef CONFIG_IPV6_SUBTREES
 755         if (rt0->fib6_src.plen)
 756                 key_plen = rt0->fib6_src.plen;
 757 #endif
 758         if (fn->fn_bit != key_plen)
 759                 return net->ipv6.fib6_null_entry;
 760
 761         match = find_rr_leaf(fn, leaf, rt0, rt0->fib6_metric, oif, strict,
 762                              &do_rr);
 763
 764         if (do_rr) {
 765                 struct fib6_info *next = rcu_dereference(rt0->fib6_next);
 766
 767                 /* no entries matched; do round-robin */
 768                 if (!next || next->fib6_metric != rt0->fib6_metric)
 769                         next = leaf;
 770
 771                 if (next != rt0) {
 772                         spin_lock_bh(&leaf->fib6_table->tb6_lock);
 773                         /* make sure next is not being deleted from the tree */
 774                         if (next->fib6_node)
 775                                 rcu_assign_pointer(fn->rr_ptr, next);
 776                         spin_unlock_bh(&leaf->fib6_table->tb6_lock);
 777                 }
 778         }
 779
 780         return match ? match : net->ipv6.fib6_null_entry;
 781 }
 782
 783 static bool rt6_is_gw_or_nonexthop(const struct fib6_info *rt)
 784 {
 785         return (rt->fib6_flags & (RTF_NONEXTHOP | RTF_GATEWAY));
 786 }
 787
 788 #ifdef CONFIG_IPV6_ROUTE_INFO
 789 int rt6_route_rcv(struct net_device *dev, u8 *opt, int len,
 790                   const struct in6_addr *gwaddr)
 791 {
 792         struct net *net = dev_net(dev);
 793         struct route_info *rinfo = (struct route_info *) opt;
 794         struct in6_addr prefix_buf, *prefix;
 795         unsigned int pref;
 796         unsigned long lifetime;
 797         struct fib6_info *rt;
 798
 799         if (len < sizeof(struct route_info)) {
 800                 return -EINVAL;
 801         }
 802
 803         /* Sanity check for prefix_len and length */
 804         if (rinfo->length > 3) {
 805                 return -EINVAL;
 806         } else if (rinfo->prefix_len > 128) {
 807                 return -EINVAL;
 808         } else if (rinfo->prefix_len > 64) {
 809                 if (rinfo->length < 2) {
 810                         return -EINVAL;
 811                 }
 812         } else if (rinfo->prefix_len > 0) {
 813                 if (rinfo->length < 1) {
 814                         return -EINVAL;
 815                 }
 816         }
 817
 818         pref = rinfo->route_pref;
 819         if (pref == ICMPV6_ROUTER_PREF_INVALID)
 820                 return -EINVAL;
 821
 822         lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ);
 823
 824         if (rinfo->length == 3)
 825                 prefix = (struct in6_addr *)rinfo->prefix;
 826         else {
 827                 /* this function is safe */
 828                 ipv6_addr_prefix(&prefix_buf,
 829                                  (struct in6_addr *)rinfo->prefix,
 830                                  rinfo->prefix_len);
 831                 prefix = &prefix_buf;
 832         }
 833
 834         if (rinfo->prefix_len == 0)
 835                 rt = rt6_get_dflt_router(net, gwaddr, dev);
 836         else
 837                 rt = rt6_get_route_info(net, prefix, rinfo->prefix_len,
 838                                         gwaddr, dev);
 839
 840         if (rt && !lifetime) {
 841                 ip6_del_rt(net, rt);
 842                 rt = NULL;
 843         }
 844
 845         if (!rt && lifetime)
 846                 rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr,
 847                                         dev, pref);
 848         else if (rt)
 849                 rt->fib6_flags = RTF_ROUTEINFO |
 850                                  (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref);
 851
 852         if (rt) {
 853                 if (!addrconf_finite_timeout(lifetime))
 854                         fib6_clean_expires(rt);
 855                 else
 856                         fib6_set_expires(rt, jiffies + HZ * lifetime);
 857
 858                 fib6_info_release(rt);
 859         }
 860         return 0;
 861 }
 862 #endif
 863
 864 /*
 865  *      Misc support functions
 866  */
 867
 868 /* called with rcu_lock held */
 869 static struct net_device *ip6_rt_get_dev_rcu(struct fib6_info *rt)
 870 {
 871         struct net_device *dev = rt->fib6_nh.nh_dev;
 872
 873         if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) {
 874                 /* for copies of local routes, dst->dev needs to be the
 875                  * device if it is a master device, the master device if
 876                  * device is enslaved, and the loopback as the default
 877                  */
 878                 if (netif_is_l3_slave(dev) &&
 879                     !rt6_need_strict(&rt->fib6_dst.addr))
 880                         dev = l3mdev_master_dev_rcu(dev);
 881                 else if (!netif_is_l3_master(dev))
 882                         dev = dev_net(dev)->loopback_dev;
 883                 /* last case is netif_is_l3_master(dev) is true in which
 884                  * case we want dev returned to be dev
 885                  */
 886         }
 887
 888         return dev;
 889 }
 890
 891 static const int fib6_prop[RTN_MAX + 1] = {
 892         [RTN_UNSPEC]    = 0,
 893         [RTN_UNICAST]   = 0,
 894         [RTN_LOCAL]     = 0,
 895         [RTN_BROADCAST] = 0,
 896         [RTN_ANYCAST]   = 0,
 897         [RTN_MULTICAST] = 0,
 898         [RTN_BLACKHOLE] = -EINVAL,
 899         [RTN_UNREACHABLE] = -EHOSTUNREACH,
 900         [RTN_PROHIBIT]  = -EACCES,
 901         [RTN_THROW]     = -EAGAIN,
 902         [RTN_NAT]       = -EINVAL,
 903         [RTN_XRESOLVE]  = -EINVAL,
 904 };
 905
 906 static int ip6_rt_type_to_error(u8 fib6_type)
 907 {
 908         return fib6_prop[fib6_type];
 909 }
 910
 911 static unsigned short fib6_info_dst_flags(struct fib6_info *rt)
 912 {
 913         unsigned short flags = 0;
 914
 915         if (rt->dst_nocount)
 916                 flags |= DST_NOCOUNT;
 917         if (rt->dst_nopolicy)
 918                 flags |= DST_NOPOLICY;
 919         if (rt->dst_host)
 920                 flags |= DST_HOST;
 921
 922         return flags;
 923 }
 924
 925 static void ip6_rt_init_dst_reject(struct rt6_info *rt, struct fib6_info *ort)
 926 {
 927         rt->dst.error = ip6_rt_type_to_error(ort->fib6_type);
 928
 929         switch (ort->fib6_type) {
 930         case RTN_BLACKHOLE:
 931                 rt->dst.output = dst_discard_out;
 932                 rt->dst.input = dst_discard;
 933                 break;
 934         case RTN_PROHIBIT:
 935                 rt->dst.output = ip6_pkt_prohibit_out;
 936                 rt->dst.input = ip6_pkt_prohibit;
 937                 break;
 938         case RTN_THROW:
 939         case RTN_UNREACHABLE:
 940         default:
 941                 rt->dst.output = ip6_pkt_discard_out;
 942                 rt->dst.input = ip6_pkt_discard;
 943                 break;
 944         }
 945 }
 946
 947 static void ip6_rt_init_dst(struct rt6_info *rt, struct fib6_info *ort)
 948 {
 949         if (ort->fib6_flags & RTF_REJECT) {
 950                 ip6_rt_init_dst_reject(rt, ort);
 951                 return;
 952         }
 953
 954         rt->dst.error = 0;
 955         rt->dst.output = ip6_output;
 956
 957         if (ort->fib6_type == RTN_LOCAL || ort->fib6_type == RTN_ANYCAST) {
 958                 rt->dst.input = ip6_input;
 959         } else if (ipv6_addr_type(&ort->fib6_dst.addr) & IPV6_ADDR_MULTICAST) {
 960                 rt->dst.input = ip6_mc_input;
 961         } else {
 962                 rt->dst.input = ip6_forward;
 963         }
 964
 965         if (ort->fib6_nh.nh_lwtstate) {
 966                 rt->dst.lwtstate = lwtstate_get(ort->fib6_nh.nh_lwtstate);
 967                 lwtunnel_set_redirect(&rt->dst);
 968         }
 969
 970         rt->dst.lastuse = jiffies;
 971 }
 972
 973 /* Caller must already hold reference to @from */
 974 static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from)
 975 {
 976         rt->rt6i_flags &= ~RTF_EXPIRES;
 977         rcu_assign_pointer(rt->from, from);
 978         ip_dst_init_metrics(&rt->dst, from->fib6_metrics);
 979 }
 980
 981 /* Caller must already hold reference to @ort */
 982 static void ip6_rt_copy_init(struct rt6_info *rt, struct fib6_info *ort)
 983 {
 984         struct net_device *dev = fib6_info_nh_dev(ort);
 985
 986         ip6_rt_init_dst(rt, ort);
 987
 988         rt->rt6i_dst = ort->fib6_dst;
 989         rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL;
 990         rt->rt6i_gateway = ort->fib6_nh.nh_gw;
 991         rt->rt6i_flags = ort->fib6_flags;
 992         rt6_set_from(rt, ort);
 993 #ifdef CONFIG_IPV6_SUBTREES
 994         rt->rt6i_src = ort->fib6_src;
 995 #endif
 996 }
 997
 998 static struct fib6_node* fib6_backtrack(struct fib6_node *fn,
 999                                         struct in6_addr *saddr)
1000 {
1001         struct fib6_node *pn, *sn;
1002         while (1) {
1003                 if (fn->fn_flags & RTN_TL_ROOT)
1004                         return NULL;
1005                 pn = rcu_dereference(fn->parent);
1006                 sn = FIB6_SUBTREE(pn);
1007                 if (sn && sn != fn)
1008                         fn = fib6_node_lookup(sn, NULL, saddr);
1009                 else
1010                         fn = pn;
1011                 if (fn->fn_flags & RTN_RTINFO)
1012                         return fn;
1013         }
1014 }
1015
1016 static bool ip6_hold_safe(struct net *net, struct rt6_info **prt,
1017                           bool null_fallback)
1018 {
1019         struct rt6_info *rt = *prt;
1020
1021         if (dst_hold_safe(&rt->dst))
1022                 return true;
1023         if (null_fallback) {
1024                 rt = net->ipv6.ip6_null_entry;
1025                 dst_hold(&rt->dst);
1026         } else {
1027                 rt = NULL;
1028         }
1029         *prt = rt;
1030         return false;
1031 }
1032
1033 /* called with rcu_lock held */
1034 static struct rt6_info *ip6_create_rt_rcu(struct fib6_info *rt)
1035 {
1036         unsigned short flags = fib6_info_dst_flags(rt);
1037         struct net_device *dev = rt->fib6_nh.nh_dev;
1038         struct rt6_info *nrt;
1039
1040         if (!fib6_info_hold_safe(rt))
1041                 return NULL;
1042
1043         nrt = ip6_dst_alloc(dev_net(dev), dev, flags);
1044         if (nrt)
1045                 ip6_rt_copy_init(nrt, rt);
1046         else
1047                 fib6_info_release(rt);
1048
1049         return nrt;
1050 }
1051
1052 static struct rt6_info *ip6_pol_route_lookup(struct net *net,
1053                                              struct fib6_table *table,
1054                                              struct flowi6 *fl6,
1055                                              const struct sk_buff *skb,
1056                                              int flags)
1057 {
1058         struct fib6_info *f6i;
1059         struct fib6_node *fn;
1060         struct rt6_info *rt;
1061
1062         if (fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF)
1063                 flags &= ~RT6_LOOKUP_F_IFACE;
1064
1065         rcu_read_lock();
1066         fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
1067 restart:
1068         f6i = rcu_dereference(fn->leaf);
1069         if (!f6i) {
1070                 f6i = net->ipv6.fib6_null_entry;
1071         } else {
1072                 f6i = rt6_device_match(net, f6i, &fl6->saddr,
1073                                       fl6->flowi6_oif, flags);
1074                 if (f6i->fib6_nsiblings && fl6->flowi6_oif == 0)
1075                         f6i = fib6_multipath_select(net, f6i, fl6,
1076                                                     fl6->flowi6_oif, skb,
1077                                                     flags);
1078         }
1079         if (f6i == net->ipv6.fib6_null_entry) {
1080                 fn = fib6_backtrack(fn, &fl6->saddr);
1081                 if (fn)
1082                         goto restart;
1083         }
1084
1085         trace_fib6_table_lookup(net, f6i, table, fl6);
1086
1087         /* Search through exception table */
1088         rt = rt6_find_cached_rt(f6i, &fl6->daddr, &fl6->saddr);
1089         if (rt) {
1090                 if (ip6_hold_safe(net, &rt, true))
1091                         dst_use_noref(&rt->dst, jiffies);
1092         } else if (f6i == net->ipv6.fib6_null_entry) {
1093                 rt = net->ipv6.ip6_null_entry;
1094                 dst_hold(&rt->dst);
1095         } else {
1096                 rt = ip6_create_rt_rcu(f6i);
1097                 if (!rt) {
1098                         rt = net->ipv6.ip6_null_entry;
1099                         dst_hold(&rt->dst);
1100                 }
1101         }
1102
1103         rcu_read_unlock();
1104
1105         return rt;
1106 }
1107
1108 struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6,
1109                                    const struct sk_buff *skb, int flags)
1110 {
1111         return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_lookup);
1112 }
1113 EXPORT_SYMBOL_GPL(ip6_route_lookup);
1114
1115 struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr,
1116                             const struct in6_addr *saddr, int oif,
1117                             const struct sk_buff *skb, int strict)
1118 {
1119         struct flowi6 fl6 = {
1120                 .flowi6_oif = oif,
1121                 .daddr = *daddr,
1122         };
1123         struct dst_entry *dst;
1124         int flags = strict ? RT6_LOOKUP_F_IFACE : 0;
1125
1126         if (saddr) {
1127                 memcpy(&fl6.saddr, saddr, sizeof(*saddr));
1128                 flags |= RT6_LOOKUP_F_HAS_SADDR;
1129         }
1130
1131         dst = fib6_rule_lookup(net, &fl6, skb, flags, ip6_pol_route_lookup);
1132         if (dst->error == 0)
1133                 return (struct rt6_info *) dst;
1134
1135         dst_release(dst);
1136
1137         return NULL;
1138 }
1139 EXPORT_SYMBOL(rt6_lookup);
1140
1141 /* ip6_ins_rt is called with FREE table->tb6_lock.
1142  * It takes new route entry, the addition fails by any reason the
1143  * route is released.
1144  * Caller must hold dst before calling it.
1145  */
1146
1147 static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info,
1148                         struct netlink_ext_ack *extack)
1149 {
1150         int err;
1151         struct fib6_table *table;
1152
1153         table = rt->fib6_table;
1154         spin_lock_bh(&table->tb6_lock);
1155         err = fib6_add(&table->tb6_root, rt, info, extack);
1156         spin_unlock_bh(&table->tb6_lock);
1157
1158         return err;
1159 }
1160
1161 int ip6_ins_rt(struct net *net, struct fib6_info *rt)
1162 {
1163         struct nl_info info = { .nl_net = net, };
1164
1165         return __ip6_ins_rt(rt, &info, NULL);
1166 }
1167
1168 static struct rt6_info *ip6_rt_cache_alloc(struct fib6_info *ort,
1169                                            const struct in6_addr *daddr,
1170                                            const struct in6_addr *saddr)
1171 {
1172         struct net_device *dev;
1173         struct rt6_info *rt;
1174
1175         /*
1176          *      Clone the route.
1177          */
1178
1179         if (!fib6_info_hold_safe(ort))
1180                 return NULL;
1181
1182         dev = ip6_rt_get_dev_rcu(ort);
1183         rt = ip6_dst_alloc(dev_net(dev), dev, 0);
1184         if (!rt) {
1185                 fib6_info_release(ort);
1186                 return NULL;
1187         }
1188
1189         ip6_rt_copy_init(rt, ort);
1190         rt->rt6i_flags |= RTF_CACHE;
1191         rt->dst.flags |= DST_HOST;
1192         rt->rt6i_dst.addr = *daddr;
1193         rt->rt6i_dst.plen = 128;
1194
1195         if (!rt6_is_gw_or_nonexthop(ort)) {
1196                 if (ort->fib6_dst.plen != 128 &&
1197                     ipv6_addr_equal(&ort->fib6_dst.addr, daddr))
1198                         rt->rt6i_flags |= RTF_ANYCAST;
1199 #ifdef CONFIG_IPV6_SUBTREES
1200                 if (rt->rt6i_src.plen && saddr) {
1201                         rt->rt6i_src.addr = *saddr;
1202                         rt->rt6i_src.plen = 128;
1203                 }
1204 #endif
1205         }
1206
1207         return rt;
1208 }
1209
1210 static struct rt6_info *ip6_rt_pcpu_alloc(struct fib6_info *rt)
1211 {
1212         unsigned short flags = fib6_info_dst_flags(rt);
1213         struct net_device *dev;
1214         struct rt6_info *pcpu_rt;
1215
1216         if (!fib6_info_hold_safe(rt))
1217                 return NULL;
1218
1219         rcu_read_lock();
1220         dev = ip6_rt_get_dev_rcu(rt);
1221         pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags);
1222         rcu_read_unlock();
1223         if (!pcpu_rt) {
1224                 fib6_info_release(rt);
1225                 return NULL;
1226         }
1227         ip6_rt_copy_init(pcpu_rt, rt);
1228         pcpu_rt->rt6i_flags |= RTF_PCPU;
1229         return pcpu_rt;
1230 }
1231
1232 /* It should be called with rcu_read_lock() acquired */
1233 static struct rt6_info *rt6_get_pcpu_route(struct fib6_info *rt)
1234 {
1235         struct rt6_info *pcpu_rt, **p;
1236
1237         p = this_cpu_ptr(rt->rt6i_pcpu);
1238         pcpu_rt = *p;
1239
1240         if (pcpu_rt)
1241                 ip6_hold_safe(NULL, &pcpu_rt, false);
1242
1243         return pcpu_rt;
1244 }
1245
1246 static struct rt6_info *rt6_make_pcpu_route(struct net *net,
1247                                             struct fib6_info *rt)
1248 {
1249         struct rt6_info *pcpu_rt, *prev, **p;
1250
1251         pcpu_rt = ip6_rt_pcpu_alloc(rt);
1252         if (!pcpu_rt) {
1253                 dst_hold(&net->ipv6.ip6_null_entry->dst);
1254                 return net->ipv6.ip6_null_entry;
1255         }
1256
1257         dst_hold(&pcpu_rt->dst);
1258         p = this_cpu_ptr(rt->rt6i_pcpu);
1259         prev = cmpxchg(p, NULL, pcpu_rt);
1260         BUG_ON(prev);
1261
1262         return pcpu_rt;
1263 }
1264
1265 /* exception hash table implementation
1266  */
1267 static DEFINE_SPINLOCK(rt6_exception_lock);
1268
1269 /* Remove rt6_ex from hash table and free the memory
1270  * Caller must hold rt6_exception_lock
1271  */
1272 static void rt6_remove_exception(struct rt6_exception_bucket *bucket,
1273                                  struct rt6_exception *rt6_ex)
1274 {
1275         struct net *net;
1276
1277         if (!bucket || !rt6_ex)
1278                 return;
1279
1280         net = dev_net(rt6_ex->rt6i->dst.dev);
1281         hlist_del_rcu(&rt6_ex->hlist);
1282         dst_release(&rt6_ex->rt6i->dst);
1283         kfree_rcu(rt6_ex, rcu);
1284         WARN_ON_ONCE(!bucket->depth);
1285         bucket->depth--;
1286         net->ipv6.rt6_stats->fib_rt_cache--;
1287 }
1288
1289 /* Remove oldest rt6_ex in bucket and free the memory
1290  * Caller must hold rt6_exception_lock
1291  */
1292 static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket)
1293 {
1294         struct rt6_exception *rt6_ex, *oldest = NULL;
1295
1296         if (!bucket)
1297                 return;
1298
1299         hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
1300                 if (!oldest || time_before(rt6_ex->stamp, oldest->stamp))
1301                         oldest = rt6_ex;
1302         }
1303         rt6_remove_exception(bucket, oldest);
1304 }
1305
1306 static u32 rt6_exception_hash(const struct in6_addr *dst,
1307                               const struct in6_addr *src)
1308 {
1309         static u32 seed __read_mostly;
1310         u32 val;
1311
1312         net_get_random_once(&seed, sizeof(seed));
1313         val = jhash(dst, sizeof(*dst), seed);
1314
1315 #ifdef CONFIG_IPV6_SUBTREES
1316         if (src)
1317                 val = jhash(src, sizeof(*src), val);
1318 #endif
1319         return hash_32(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT);
1320 }
1321
1322 /* Helper function to find the cached rt in the hash table
1323  * and update bucket pointer to point to the bucket for this
1324  * (daddr, saddr) pair
1325  * Caller must hold rt6_exception_lock
1326  */
1327 static struct rt6_exception *
1328 __rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket,
1329                               const struct in6_addr *daddr,
1330                               const struct in6_addr *saddr)
1331 {
1332         struct rt6_exception *rt6_ex;
1333         u32 hval;
1334
1335         if (!(*bucket) || !daddr)
1336                 return NULL;
1337
1338         hval = rt6_exception_hash(daddr, saddr);
1339         *bucket += hval;
1340
1341         hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) {
1342                 struct rt6_info *rt6 = rt6_ex->rt6i;
1343                 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr);
1344
1345 #ifdef CONFIG_IPV6_SUBTREES
1346                 if (matched && saddr)
1347                         matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
1348 #endif
1349                 if (matched)
1350                         return rt6_ex;
1351         }
1352         return NULL;
1353 }
1354
1355 /* Helper function to find the cached rt in the hash table
1356  * and update bucket pointer to point to the bucket for this
1357  * (daddr, saddr) pair
1358  * Caller must hold rcu_read_lock()
1359  */
1360 static struct rt6_exception *
1361 __rt6_find_exception_rcu(struct rt6_exception_bucket **bucket,
1362                          const struct in6_addr *daddr,
1363                          const struct in6_addr *saddr)
1364 {
1365         struct rt6_exception *rt6_ex;
1366         u32 hval;
1367
1368         WARN_ON_ONCE(!rcu_read_lock_held());
1369
1370         if (!(*bucket) || !daddr)
1371                 return NULL;
1372
1373         hval = rt6_exception_hash(daddr, saddr);
1374         *bucket += hval;
1375
1376         hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) {
1377                 struct rt6_info *rt6 = rt6_ex->rt6i;
1378                 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr);
1379
1380 #ifdef CONFIG_IPV6_SUBTREES
1381                 if (matched && saddr)
1382                         matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
1383 #endif
1384                 if (matched)
1385                         return rt6_ex;
1386         }
1387         return NULL;
1388 }
1389
1390 static unsigned int fib6_mtu(const struct fib6_info *rt)
1391 {
1392         unsigned int mtu;
1393
1394         if (rt->fib6_pmtu) {
1395                 mtu = rt->fib6_pmtu;
1396         } else {
1397                 struct net_device *dev = fib6_info_nh_dev(rt);
1398                 struct inet6_dev *idev;
1399
1400                 rcu_read_lock();
1401                 idev = __in6_dev_get(dev);
1402                 mtu = idev->cnf.mtu6;
1403                 rcu_read_unlock();
1404         }
1405
1406         mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
1407
1408         return mtu - lwtunnel_headroom(rt->fib6_nh.nh_lwtstate, mtu);
1409 }
1410
1411 static int rt6_insert_exception(struct rt6_info *nrt,
1412                                 struct fib6_info *ort)
1413 {
1414         struct net *net = dev_net(nrt->dst.dev);
1415         struct rt6_exception_bucket *bucket;
1416         struct in6_addr *src_key = NULL;
1417         struct rt6_exception *rt6_ex;
1418         int err = 0;
1419
1420         spin_lock_bh(&rt6_exception_lock);
1421
1422         if (ort->exception_bucket_flushed) {
1423                 err = -EINVAL;
1424                 goto out;
1425         }
1426
1427         bucket = rcu_dereference_protected(ort->rt6i_exception_bucket,
1428                                         lockdep_is_held(&rt6_exception_lock));
1429         if (!bucket) {
1430                 bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket),
1431                                  GFP_ATOMIC);
1432                 if (!bucket) {
1433                         err = -ENOMEM;
1434                         goto out;
1435                 }
1436                 rcu_assign_pointer(ort->rt6i_exception_bucket, bucket);
1437         }
1438
1439 #ifdef CONFIG_IPV6_SUBTREES
1440         /* rt6i_src.plen != 0 indicates ort is in subtree
1441          * and exception table is indexed by a hash of
1442          * both rt6i_dst and rt6i_src.
1443          * Otherwise, the exception table is indexed by
1444          * a hash of only rt6i_dst.
1445          */
1446         if (ort->fib6_src.plen)
1447                 src_key = &nrt->rt6i_src.addr;
1448 #endif
1449         /* rt6_mtu_change() might lower mtu on ort.
1450          * Only insert this exception route if its mtu
1451          * is less than ort's mtu value.
1452          */
1453         if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(ort)) {
1454                 err = -EINVAL;
1455                 goto out;
1456         }
1457
1458         rt6_ex = __rt6_find_exception_spinlock(&bucket, &nrt->rt6i_dst.addr,
1459                                                src_key);
1460         if (rt6_ex)
1461                 rt6_remove_exception(bucket, rt6_ex);
1462
1463         rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC);
1464         if (!rt6_ex) {
1465                 err = -ENOMEM;
1466                 goto out;
1467         }
1468         rt6_ex->rt6i = nrt;
1469         rt6_ex->stamp = jiffies;
1470         hlist_add_head_rcu(&rt6_ex->hlist, &bucket->chain);
1471         bucket->depth++;
1472         net->ipv6.rt6_stats->fib_rt_cache++;
1473
1474         if (bucket->depth > FIB6_MAX_DEPTH)
1475                 rt6_exception_remove_oldest(bucket);
1476
1477 out:
1478         spin_unlock_bh(&rt6_exception_lock);
1479
1480         /* Update fn->fn_sernum to invalidate all cached dst */
1481         if (!err) {
1482                 spin_lock_bh(&ort->fib6_table->tb6_lock);
1483                 fib6_update_sernum(net, ort);
1484                 spin_unlock_bh(&ort->fib6_table->tb6_lock);
1485                 fib6_force_start_gc(net);
1486         }
1487
1488         return err;
1489 }
1490
1491 void rt6_flush_exceptions(struct fib6_info *rt)
1492 {
1493         struct rt6_exception_bucket *bucket;
1494         struct rt6_exception *rt6_ex;
1495         struct hlist_node *tmp;
1496         int i;
1497
1498         spin_lock_bh(&rt6_exception_lock);
1499         /* Prevent rt6_insert_exception() to recreate the bucket list */
1500         rt->exception_bucket_flushed = 1;
1501
1502         bucket = rcu_dereference_protected(rt->rt6i_exception_bucket,
1503                                     lockdep_is_held(&rt6_exception_lock));
1504         if (!bucket)
1505                 goto out;
1506
1507         for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
1508                 hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist)
1509                         rt6_remove_exception(bucket, rt6_ex);
1510                 WARN_ON_ONCE(bucket->depth);
1511                 bucket++;
1512         }
1513
1514 out:
1515         spin_unlock_bh(&rt6_exception_lock);
1516 }
1517
1518 /* Find cached rt in the hash table inside passed in rt
1519  * Caller has to hold rcu_read_lock()
1520  */
1521 static struct rt6_info *rt6_find_cached_rt(struct fib6_info *rt,
1522                                            struct in6_addr *daddr,
1523                                            struct in6_addr *saddr)
1524 {
1525         struct rt6_exception_bucket *bucket;
1526         struct in6_addr *src_key = NULL;
1527         struct rt6_exception *rt6_ex;
1528         struct rt6_info *res = NULL;
1529
1530         bucket = rcu_dereference(rt->rt6i_exception_bucket);
1531
1532 #ifdef CONFIG_IPV6_SUBTREES
1533         /* rt6i_src.plen != 0 indicates rt is in subtree
1534          * and exception table is indexed by a hash of
1535          * both rt6i_dst and rt6i_src.
1536          * Otherwise, the exception table is indexed by
1537          * a hash of only rt6i_dst.
1538          */
1539         if (rt->fib6_src.plen)
1540                 src_key = saddr;
1541 #endif
1542         rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key);
1543
1544         if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i))
1545                 res = rt6_ex->rt6i;
1546
1547         return res;
1548 }
1549
1550 /* Remove the passed in cached rt from the hash table that contains it */
1551 static int rt6_remove_exception_rt(struct rt6_info *rt)
1552 {
1553         struct rt6_exception_bucket *bucket;
1554         struct in6_addr *src_key = NULL;
1555         struct rt6_exception *rt6_ex;
1556         struct fib6_info *from;
1557         int err;
1558
1559         from = rcu_dereference(rt->from);
1560         if (!from ||
1561             !(rt->rt6i_flags & RTF_CACHE))
1562                 return -EINVAL;
1563
1564         if (!rcu_access_pointer(from->rt6i_exception_bucket))
1565                 return -ENOENT;
1566
1567         spin_lock_bh(&rt6_exception_lock);
1568         bucket = rcu_dereference_protected(from->rt6i_exception_bucket,
1569                                     lockdep_is_held(&rt6_exception_lock));
1570 #ifdef CONFIG_IPV6_SUBTREES
1571         /* rt6i_src.plen != 0 indicates 'from' is in subtree
1572          * and exception table is indexed by a hash of
1573          * both rt6i_dst and rt6i_src.
1574          * Otherwise, the exception table is indexed by
1575          * a hash of only rt6i_dst.
1576          */
1577         if (from->fib6_src.plen)
1578                 src_key = &rt->rt6i_src.addr;
1579 #endif
1580         rt6_ex = __rt6_find_exception_spinlock(&bucket,
1581                                                &rt->rt6i_dst.addr,
1582                                                src_key);
1583         if (rt6_ex) {
1584                 rt6_remove_exception(bucket, rt6_ex);
1585                 err = 0;
1586         } else {
1587                 err = -ENOENT;
1588         }
1589
1590         spin_unlock_bh(&rt6_exception_lock);
1591         return err;
1592 }
1593
1594 /* Find rt6_ex which contains the passed in rt cache and
1595  * refresh its stamp
1596  */
1597 static void rt6_update_exception_stamp_rt(struct rt6_info *rt)
1598 {
1599         struct rt6_exception_bucket *bucket;
1600         struct fib6_info *from = rt->from;
1601         struct in6_addr *src_key = NULL;
1602         struct rt6_exception *rt6_ex;
1603
1604         if (!from ||
1605             !(rt->rt6i_flags & RTF_CACHE))
1606                 return;
1607
1608         rcu_read_lock();
1609         bucket = rcu_dereference(from->rt6i_exception_bucket);
1610
1611 #ifdef CONFIG_IPV6_SUBTREES
1612         /* rt6i_src.plen != 0 indicates 'from' is in subtree
1613          * and exception table is indexed by a hash of
1614          * both rt6i_dst and rt6i_src.
1615          * Otherwise, the exception table is indexed by
1616          * a hash of only rt6i_dst.
1617          */
1618         if (from->fib6_src.plen)
1619                 src_key = &rt->rt6i_src.addr;
1620 #endif
1621         rt6_ex = __rt6_find_exception_rcu(&bucket,
1622                                           &rt->rt6i_dst.addr,
1623                                           src_key);
1624         if (rt6_ex)
1625                 rt6_ex->stamp = jiffies;
1626
1627         rcu_read_unlock();
1628 }
1629
1630 static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev,
1631                                          struct rt6_info *rt, int mtu)
1632 {
1633         /* If the new MTU is lower than the route PMTU, this new MTU will be the
1634          * lowest MTU in the path: always allow updating the route PMTU to
1635          * reflect PMTU decreases.
1636          *
1637          * If the new MTU is higher, and the route PMTU is equal to the local
1638          * MTU, this means the old MTU is the lowest in the path, so allow
1639          * updating it: if other nodes now have lower MTUs, PMTU discovery will
1640          * handle this.
1641          */
1642
1643         if (dst_mtu(&rt->dst) >= mtu)
1644                 return true;
1645
1646         if (dst_mtu(&rt->dst) == idev->cnf.mtu6)
1647                 return true;
1648
1649         return false;
1650 }
1651
1652 static void rt6_exceptions_update_pmtu(struct inet6_dev *idev,
1653                                        struct fib6_info *rt, int mtu)
1654 {
1655         struct rt6_exception_bucket *bucket;
1656         struct rt6_exception *rt6_ex;
1657         int i;
1658
1659         bucket = rcu_dereference_protected(rt->rt6i_exception_bucket,
1660                                         lockdep_is_held(&rt6_exception_lock));
1661
1662         if (!bucket)
1663                 return;
1664
1665         for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
1666                 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
1667                         struct rt6_info *entry = rt6_ex->rt6i;
1668
1669                         /* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected
1670                          * route), the metrics of its rt->from have already
1671                          * been updated.
1672                          */
1673                         if (dst_metric_raw(&entry->dst, RTAX_MTU) &&
1674                             rt6_mtu_change_route_allowed(idev, entry, mtu))
1675                                 dst_metric_set(&entry->dst, RTAX_MTU, mtu);
1676                 }
1677                 bucket++;
1678         }
1679 }
1680
1681 #define RTF_CACHE_GATEWAY       (RTF_GATEWAY | RTF_CACHE)
1682
1683 static void rt6_exceptions_clean_tohost(struct fib6_info *rt,
1684                                         struct in6_addr *gateway)
1685 {
1686         struct rt6_exception_bucket *bucket;
1687         struct rt6_exception *rt6_ex;
1688         struct hlist_node *tmp;
1689         int i;
1690
1691         if (!rcu_access_pointer(rt->rt6i_exception_bucket))
1692                 return;
1693
1694         spin_lock_bh(&rt6_exception_lock);
1695         bucket = rcu_dereference_protected(rt->rt6i_exception_bucket,
1696                                      lockdep_is_held(&rt6_exception_lock));
1697
1698         if (bucket) {
1699                 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
1700                         hlist_for_each_entry_safe(rt6_ex, tmp,
1701                                                   &bucket->chain, hlist) {
1702                                 struct rt6_info *entry = rt6_ex->rt6i;
1703
1704                                 if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) ==
1705                                     RTF_CACHE_GATEWAY &&
1706                                     ipv6_addr_equal(gateway,
1707                                                     &entry->rt6i_gateway)) {
1708                                         rt6_remove_exception(bucket, rt6_ex);
1709                                 }
1710                         }
1711                         bucket++;
1712                 }
1713         }
1714
1715         spin_unlock_bh(&rt6_exception_lock);
1716 }
1717
1718 static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket,
1719                                       struct rt6_exception *rt6_ex,
1720                                       struct fib6_gc_args *gc_args,
1721                                       unsigned long now)
1722 {
1723         struct rt6_info *rt = rt6_ex->rt6i;
1724
1725         /* we are pruning and obsoleting aged-out and non gateway exceptions
1726          * even if others have still references to them, so that on next
1727          * dst_check() such references can be dropped.
1728          * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when
1729          * expired, independently from their aging, as per RFC 8201 section 4
1730          */
1731         if (!(rt->rt6i_flags & RTF_EXPIRES)) {
1732                 if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout)) {
1733                         RT6_TRACE("aging clone %p\n", rt);
1734                         rt6_remove_exception(bucket, rt6_ex);
1735                         return;
1736                 }
1737         } else if (time_after(jiffies, rt->dst.expires)) {
1738                 RT6_TRACE("purging expired route %p\n", rt);
1739                 rt6_remove_exception(bucket, rt6_ex);
1740                 return;
1741         }
1742
1743         if (rt->rt6i_flags & RTF_GATEWAY) {
1744                 struct neighbour *neigh;
1745                 __u8 neigh_flags = 0;
1746
1747                 neigh = __ipv6_neigh_lookup_noref(rt->dst.dev, &rt->rt6i_gateway);
1748                 if (neigh)
1749                         neigh_flags = neigh->flags;
1750
1751                 if (!(neigh_flags & NTF_ROUTER)) {
1752                         RT6_TRACE("purging route %p via non-router but gateway\n",
1753                                   rt);
1754                         rt6_remove_exception(bucket, rt6_ex);
1755                         return;
1756                 }
1757         }
1758
1759         gc_args->more++;
1760 }
1761
1762 void rt6_age_exceptions(struct fib6_info *rt,
1763                         struct fib6_gc_args *gc_args,
1764                         unsigned long now)
1765 {
1766         struct rt6_exception_bucket *bucket;
1767         struct rt6_exception *rt6_ex;
1768         struct hlist_node *tmp;
1769         int i;
1770
1771         if (!rcu_access_pointer(rt->rt6i_exception_bucket))
1772                 return;
1773
1774         rcu_read_lock_bh();
1775         spin_lock(&rt6_exception_lock);
1776         bucket = rcu_dereference_protected(rt->rt6i_exception_bucket,
1777                                     lockdep_is_held(&rt6_exception_lock));
1778
1779         if (bucket) {
1780                 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
1781                         hlist_for_each_entry_safe(rt6_ex, tmp,
1782                                                   &bucket->chain, hlist) {
1783                                 rt6_age_examine_exception(bucket, rt6_ex,
1784                                                           gc_args, now);
1785                         }
1786                         bucket++;
1787                 }
1788         }
1789         spin_unlock(&rt6_exception_lock);
1790         rcu_read_unlock_bh();
1791 }
1792
1793 /* must be called with rcu lock held */
1794 struct fib6_info *fib6_table_lookup(struct net *net, struct fib6_table *table,
1795                                     int oif, struct flowi6 *fl6, int strict)
1796 {
1797         struct fib6_node *fn, *saved_fn;
1798         struct fib6_info *f6i;
1799
1800         fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
1801         saved_fn = fn;
1802
1803         if (fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF)
1804                 oif = 0;
1805
1806 redo_rt6_select:
1807         f6i = rt6_select(net, fn, oif, strict);
1808         if (f6i == net->ipv6.fib6_null_entry) {
1809                 fn = fib6_backtrack(fn, &fl6->saddr);
1810                 if (fn)
1811                         goto redo_rt6_select;
1812                 else if (strict & RT6_LOOKUP_F_REACHABLE) {
1813                         /* also consider unreachable route */
1814                         strict &= ~RT6_LOOKUP_F_REACHABLE;
1815                         fn = saved_fn;
1816                         goto redo_rt6_select;
1817                 }
1818         }
1819
1820         trace_fib6_table_lookup(net, f6i, table, fl6);
1821
1822         return f6i;
1823 }
1824
1825 struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
1826                                int oif, struct flowi6 *fl6,
1827                                const struct sk_buff *skb, int flags)
1828 {
1829         struct fib6_info *f6i;
1830         struct rt6_info *rt;
1831         int strict = 0;
1832
1833         strict |= flags & RT6_LOOKUP_F_IFACE;
1834         strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE;
1835         if (net->ipv6.devconf_all->forwarding == 0)
1836                 strict |= RT6_LOOKUP_F_REACHABLE;
1837
1838         rcu_read_lock();
1839
1840         f6i = fib6_table_lookup(net, table, oif, fl6, strict);
1841         if (f6i->fib6_nsiblings)
1842                 f6i = fib6_multipath_select(net, f6i, fl6, oif, skb, strict);
1843
1844         if (f6i == net->ipv6.fib6_null_entry) {
1845                 rt = net->ipv6.ip6_null_entry;
1846                 rcu_read_unlock();
1847                 dst_hold(&rt->dst);
1848                 return rt;
1849         }
1850
1851         /*Search through exception table */
1852         rt = rt6_find_cached_rt(f6i, &fl6->daddr, &fl6->saddr);
1853         if (rt) {
1854                 if (ip6_hold_safe(net, &rt, true))
1855                         dst_use_noref(&rt->dst, jiffies);
1856
1857                 rcu_read_unlock();
1858                 return rt;
1859         } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) &&
1860                             !(f6i->fib6_flags & RTF_GATEWAY))) {
1861                 /* Create a RTF_CACHE clone which will not be
1862                  * owned by the fib6 tree.  It is for the special case where
1863                  * the daddr in the skb during the neighbor look-up is different
1864                  * from the fl6->daddr used to look-up route here.
1865                  */
1866                 struct rt6_info *uncached_rt;
1867
1868                 uncached_rt = ip6_rt_cache_alloc(f6i, &fl6->daddr, NULL);
1869
1870                 rcu_read_unlock();
1871
1872                 if (uncached_rt) {
1873                         /* Uncached_rt's refcnt is taken during ip6_rt_cache_alloc()
1874                          * No need for another dst_hold()
1875                          */
1876                         rt6_uncached_list_add(uncached_rt);
1877                         atomic_inc(&net->ipv6.rt6_stats->fib_rt_uncache);
1878                 } else {
1879                         uncached_rt = net->ipv6.ip6_null_entry;
1880                         dst_hold(&uncached_rt->dst);
1881                 }
1882
1883                 return uncached_rt;
1884         } else {
1885                 /* Get a percpu copy */
1886
1887                 struct rt6_info *pcpu_rt;
1888
1889                 local_bh_disable();
1890                 pcpu_rt = rt6_get_pcpu_route(f6i);
1891
1892                 if (!pcpu_rt)
1893                         pcpu_rt = rt6_make_pcpu_route(net, f6i);
1894
1895                 local_bh_enable();
1896                 rcu_read_unlock();
1897
1898                 return pcpu_rt;
1899         }
1900 }
1901 EXPORT_SYMBOL_GPL(ip6_pol_route);
1902
1903 static struct rt6_info *ip6_pol_route_input(struct net *net,
1904                                             struct fib6_table *table,
1905                                             struct flowi6 *fl6,
1906                                             const struct sk_buff *skb,
1907                                             int flags)
1908 {
1909         return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags);
1910 }
1911
1912 struct dst_entry *ip6_route_input_lookup(struct net *net,
1913                                          struct net_device *dev,
1914                                          struct flowi6 *fl6,
1915                                          const struct sk_buff *skb,
1916                                          int flags)
1917 {
1918         if (rt6_need_strict(&fl6->daddr) && dev->type != ARPHRD_PIMREG)
1919                 flags |= RT6_LOOKUP_F_IFACE;
1920
1921         return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_input);
1922 }
1923 EXPORT_SYMBOL_GPL(ip6_route_input_lookup);
1924
1925 static void ip6_multipath_l3_keys(const struct sk_buff *skb,
1926                                   struct flow_keys *keys,
1927                                   struct flow_keys *flkeys)
1928 {
1929         const struct ipv6hdr *outer_iph = ipv6_hdr(skb);
1930         const struct ipv6hdr *key_iph = outer_iph;
1931         struct flow_keys *_flkeys = flkeys;
1932         const struct ipv6hdr *inner_iph;
1933         const struct icmp6hdr *icmph;
1934         struct ipv6hdr _inner_iph;
1935         struct icmp6hdr _icmph;
1936
1937         if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6))
1938                 goto out;
1939
1940         icmph = skb_header_pointer(skb, skb_transport_offset(skb),
1941                                    sizeof(_icmph), &_icmph);
1942         if (!icmph)
1943                 goto out;
1944
1945         if (icmph->icmp6_type != ICMPV6_DEST_UNREACH &&
1946             icmph->icmp6_type != ICMPV6_PKT_TOOBIG &&
1947             icmph->icmp6_type != ICMPV6_TIME_EXCEED &&
1948             icmph->icmp6_type != ICMPV6_PARAMPROB)
1949                 goto out;
1950
1951         inner_iph = skb_header_pointer(skb,
1952                                        skb_transport_offset(skb) + sizeof(*icmph),
1953                                        sizeof(_inner_iph), &_inner_iph);
1954         if (!inner_iph)
1955                 goto out;
1956
1957         key_iph = inner_iph;
1958         _flkeys = NULL;
1959 out:
1960         if (_flkeys) {
1961                 keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src;
1962                 keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst;
1963                 keys->tags.flow_label = _flkeys->tags.flow_label;
1964                 keys->basic.ip_proto = _flkeys->basic.ip_proto;
1965         } else {
1966                 keys->addrs.v6addrs.src = key_iph->saddr;
1967                 keys->addrs.v6addrs.dst = key_iph->daddr;
1968                 keys->tags.flow_label = ip6_flowlabel(key_iph);
1969                 keys->basic.ip_proto = key_iph->nexthdr;
1970         }
1971 }
1972
1973 /* if skb is set it will be used and fl6 can be NULL */
1974 u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6,
1975                        const struct sk_buff *skb, struct flow_keys *flkeys)
1976 {
1977         struct flow_keys hash_keys;
1978         u32 mhash;
1979
1980         switch (ip6_multipath_hash_policy(net)) {
1981         case 0:
1982                 memset(&hash_keys, 0, sizeof(hash_keys));
1983                 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1984                 if (skb) {
1985                         ip6_multipath_l3_keys(skb, &hash_keys, flkeys);
1986                 } else {
1987                         hash_keys.addrs.v6addrs.src = fl6->saddr;
1988                         hash_keys.addrs.v6addrs.dst = fl6->daddr;
1989                         hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
1990                         hash_keys.basic.ip_proto = fl6->flowi6_proto;
1991                 }
1992                 break;
1993         case 1:
1994                 if (skb) {
1995                         unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
1996                         struct flow_keys keys;
1997
1998                         /* short-circuit if we already have L4 hash present */
1999                         if (skb->l4_hash)
2000                                 return skb_get_hash_raw(skb) >> 1;
2001
2002                         memset(&hash_keys, 0, sizeof(hash_keys));
2003
2004                         if (!flkeys) {
2005                                 skb_flow_dissect_flow_keys(skb, &keys, flag);
2006                                 flkeys = &keys;
2007                         }
2008                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2009                         hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src;
2010                         hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst;
2011                         hash_keys.ports.src = flkeys->ports.src;
2012                         hash_keys.ports.dst = flkeys->ports.dst;
2013                         hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
2014                 } else {
2015                         memset(&hash_keys, 0, sizeof(hash_keys));
2016                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2017                         hash_keys.addrs.v6addrs.src = fl6->saddr;
2018                         hash_keys.addrs.v6addrs.dst = fl6->daddr;
2019                         hash_keys.ports.src = fl6->fl6_sport;
2020                         hash_keys.ports.dst = fl6->fl6_dport;
2021                         hash_keys.basic.ip_proto = fl6->flowi6_proto;
2022                 }
2023                 break;
2024         }
2025         mhash = flow_hash_from_keys(&hash_keys);
2026
2027         return mhash >> 1;
2028 }
2029
2030 void ip6_route_input(struct sk_buff *skb)
2031 {
2032         const struct ipv6hdr *iph = ipv6_hdr(skb);
2033         struct net *net = dev_net(skb->dev);
2034         int flags = RT6_LOOKUP_F_HAS_SADDR;
2035         struct ip_tunnel_info *tun_info;
2036         struct flowi6 fl6 = {
2037                 .flowi6_iif = skb->dev->ifindex,
2038                 .daddr = iph->daddr,
2039                 .saddr = iph->saddr,
2040                 .flowlabel = ip6_flowinfo(iph),
2041                 .flowi6_mark = skb->mark,
2042                 .flowi6_proto = iph->nexthdr,
2043         };
2044         struct flow_keys *flkeys = NULL, _flkeys;
2045
2046         tun_info = skb_tunnel_info(skb);
2047         if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
2048                 fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id;
2049
2050         if (fib6_rules_early_flow_dissect(net, skb, &fl6, &_flkeys))
2051                 flkeys = &_flkeys;
2052
2053         if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6))
2054                 fl6.mp_hash = rt6_multipath_hash(net, &fl6, skb, flkeys);
2055         skb_dst_drop(skb);
2056         skb_dst_set(skb,
2057                     ip6_route_input_lookup(net, skb->dev, &fl6, skb, flags));
2058 }
2059
2060 static struct rt6_info *ip6_pol_route_output(struct net *net,
2061                                              struct fib6_table *table,
2062                                              struct flowi6 *fl6,
2063                                              const struct sk_buff *skb,
2064                                              int flags)
2065 {
2066         return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags);
2067 }
2068
2069 struct dst_entry *ip6_route_output_flags(struct net *net, const struct sock *sk,
2070                                          struct flowi6 *fl6, int flags)
2071 {
2072         bool any_src;
2073
2074         if (ipv6_addr_type(&fl6->daddr) &
2075             (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL)) {
2076                 struct dst_entry *dst;
2077
2078                 dst = l3mdev_link_scope_lookup(net, fl6);
2079                 if (dst)
2080                         return dst;
2081         }
2082
2083         fl6->flowi6_iif = LOOPBACK_IFINDEX;
2084
2085         any_src = ipv6_addr_any(&fl6->saddr);
2086         if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) ||
2087             (fl6->flowi6_oif && any_src))
2088                 flags |= RT6_LOOKUP_F_IFACE;
2089
2090         if (!any_src)
2091                 flags |= RT6_LOOKUP_F_HAS_SADDR;
2092         else if (sk)
2093                 flags |= rt6_srcprefs2flags(inet6_sk(sk)->srcprefs);
2094
2095         return fib6_rule_lookup(net, fl6, NULL, flags, ip6_pol_route_output);
2096 }
2097 EXPORT_SYMBOL_GPL(ip6_route_output_flags);
2098
2099 struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2100 {
2101         struct rt6_info *rt, *ort = (struct rt6_info *) dst_orig;
2102         struct net_device *loopback_dev = net->loopback_dev;
2103         struct dst_entry *new = NULL;
2104
2105         rt = dst_alloc(&ip6_dst_blackhole_ops, loopback_dev, 1,
2106                        DST_OBSOLETE_DEAD, 0);
2107         if (rt) {
2108                 rt6_info_init(rt);
2109                 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc);
2110
2111                 new = &rt->dst;
2112                 new->__use = 1;
2113                 new->input = dst_discard;
2114                 new->output = dst_discard_out;
2115
2116                 dst_copy_metrics(new, &ort->dst);
2117
2118                 rt->rt6i_idev = in6_dev_get(loopback_dev);
2119                 rt->rt6i_gateway = ort->rt6i_gateway;
2120                 rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU;
2121
2122                 memcpy(&rt->rt6i_dst, &ort->rt6i_dst, sizeof(struct rt6key));
2123 #ifdef CONFIG_IPV6_SUBTREES
2124                 memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key));
2125 #endif
2126         }
2127
2128         dst_release(dst_orig);
2129         return new ? new : ERR_PTR(-ENOMEM);
2130 }
2131
2132 /*
2133  *      Destination cache support functions
2134  */
2135
2136 static bool fib6_check(struct fib6_info *f6i, u32 cookie)
2137 {
2138         u32 rt_cookie = 0;
2139
2140         if (!fib6_get_cookie_safe(f6i, &rt_cookie) || rt_cookie != cookie)
2141                 return false;
2142
2143         if (fib6_check_expired(f6i))
2144                 return false;
2145
2146         return true;
2147 }
2148
2149 static struct dst_entry *rt6_check(struct rt6_info *rt,
2150                                    struct fib6_info *from,
2151                                    u32 cookie)
2152 {
2153         u32 rt_cookie = 0;
2154
2155         if ((from && !fib6_get_cookie_safe(from, &rt_cookie)) ||
2156             rt_cookie != cookie)
2157                 return NULL;
2158
2159         if (rt6_check_expired(rt))
2160                 return NULL;
2161
2162         return &rt->dst;
2163 }
2164
2165 static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt,
2166                                             struct fib6_info *from,
2167                                             u32 cookie)
2168 {
2169         if (!__rt6_check_expired(rt) &&
2170             rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
2171             fib6_check(from, cookie))
2172                 return &rt->dst;
2173         else
2174                 return NULL;
2175 }
2176
2177 static struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie)
2178 {
2179         struct dst_entry *dst_ret;
2180         struct fib6_info *from;
2181         struct rt6_info *rt;
2182
2183         rt = container_of(dst, struct rt6_info, dst);
2184
2185         rcu_read_lock();
2186
2187         /* All IPV6 dsts are created with ->obsolete set to the value
2188          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
2189          * into this function always.
2190          */
2191
2192         from = rcu_dereference(rt->from);
2193
2194         if (from && (rt->rt6i_flags & RTF_PCPU ||
2195             unlikely(!list_empty(&rt->rt6i_uncached))))
2196                 dst_ret = rt6_dst_from_check(rt, from, cookie);
2197         else
2198                 dst_ret = rt6_check(rt, from, cookie);
2199
2200         rcu_read_unlock();
2201
2202         return dst_ret;
2203 }
2204
2205 static struct dst_entry *ip6_negative_advice(struct dst_entry *dst)
2206 {
2207         struct rt6_info *rt = (struct rt6_info *) dst;
2208
2209         if (rt) {
2210                 if (rt->rt6i_flags & RTF_CACHE) {
2211                         rcu_read_lock();
2212                         if (rt6_check_expired(rt)) {
2213                                 rt6_remove_exception_rt(rt);
2214                                 dst = NULL;
2215                         }
2216                         rcu_read_unlock();
2217                 } else {
2218                         dst_release(dst);
2219                         dst = NULL;
2220                 }
2221         }
2222         return dst;
2223 }
2224
2225 static void ip6_link_failure(struct sk_buff *skb)
2226 {
2227         struct rt6_info *rt;
2228
2229         icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
2230
2231         rt = (struct rt6_info *) skb_dst(skb);
2232         if (rt) {
2233                 rcu_read_lock();
2234                 if (rt->rt6i_flags & RTF_CACHE) {
2235                         if (dst_hold_safe(&rt->dst))
2236                                 rt6_remove_exception_rt(rt);
2237                 } else {
2238                         struct fib6_info *from;
2239                         struct fib6_node *fn;
2240
2241                         from = rcu_dereference(rt->from);
2242                         if (from) {
2243                                 fn = rcu_dereference(from->fib6_node);
2244                                 if (fn && (rt->rt6i_flags & RTF_DEFAULT))
2245                                         fn->fn_sernum = -1;
2246                         }
2247                 }
2248                 rcu_read_unlock();
2249         }
2250 }
2251
2252 static void rt6_update_expires(struct rt6_info *rt0, int timeout)
2253 {
2254         if (!(rt0->rt6i_flags & RTF_EXPIRES)) {
2255                 struct fib6_info *from;
2256
2257                 rcu_read_lock();
2258                 from = rcu_dereference(rt0->from);
2259                 if (from)
2260                         rt0->dst.expires = from->expires;
2261                 rcu_read_unlock();
2262         }
2263
2264         dst_set_expires(&rt0->dst, timeout);
2265         rt0->rt6i_flags |= RTF_EXPIRES;
2266 }
2267
2268 static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu)
2269 {
2270         struct net *net = dev_net(rt->dst.dev);
2271
2272         dst_metric_set(&rt->dst, RTAX_MTU, mtu);
2273         rt->rt6i_flags |= RTF_MODIFIED;
2274         rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires);
2275 }
2276
2277 static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt)
2278 {
2279         bool from_set;
2280
2281         rcu_read_lock();
2282         from_set = !!rcu_dereference(rt->from);
2283         rcu_read_unlock();
2284
2285         return !(rt->rt6i_flags & RTF_CACHE) &&
2286                 (rt->rt6i_flags & RTF_PCPU || from_set);
2287 }
2288
2289 static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk,
2290                                  const struct ipv6hdr *iph, u32 mtu)
2291 {
2292         const struct in6_addr *daddr, *saddr;
2293         struct rt6_info *rt6 = (struct rt6_info *)dst;
2294
2295         if (dst_metric_locked(dst, RTAX_MTU))
2296                 return;
2297
2298         if (iph) {
2299                 daddr = &iph->daddr;
2300                 saddr = &iph->saddr;
2301         } else if (sk) {
2302                 daddr = &sk->sk_v6_daddr;
2303                 saddr = &inet6_sk(sk)->saddr;
2304         } else {
2305                 daddr = NULL;
2306                 saddr = NULL;
2307         }
2308         dst_confirm_neigh(dst, daddr);
2309         mtu = max_t(u32, mtu, IPV6_MIN_MTU);
2310         if (mtu >= dst_mtu(dst))
2311                 return;
2312
2313         if (!rt6_cache_allowed_for_pmtu(rt6)) {
2314                 rt6_do_update_pmtu(rt6, mtu);
2315                 /* update rt6_ex->stamp for cache */
2316                 if (rt6->rt6i_flags & RTF_CACHE)
2317                         rt6_update_exception_stamp_rt(rt6);
2318         } else if (daddr) {
2319                 struct fib6_info *from;
2320                 struct rt6_info *nrt6;
2321
2322                 rcu_read_lock();
2323                 from = rcu_dereference(rt6->from);
2324                 nrt6 = ip6_rt_cache_alloc(from, daddr, saddr);
2325                 if (nrt6) {
2326                         rt6_do_update_pmtu(nrt6, mtu);
2327                         if (rt6_insert_exception(nrt6, from))
2328                                 dst_release_immediate(&nrt6->dst);
2329                 }
2330                 rcu_read_unlock();
2331         }
2332 }
2333
2334 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
2335                                struct sk_buff *skb, u32 mtu)
2336 {
2337         __ip6_rt_update_pmtu(dst, sk, skb ? ipv6_hdr(skb) : NULL, mtu);
2338 }
2339
2340 void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu,
2341                      int oif, u32 mark, kuid_t uid)
2342 {
2343         const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
2344         struct dst_entry *dst;
2345         struct flowi6 fl6 = {
2346                 .flowi6_oif = oif,
2347                 .flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark),
2348                 .daddr = iph->daddr,
2349                 .saddr = iph->saddr,
2350                 .flowlabel = ip6_flowinfo(iph),
2351                 .flowi6_uid = uid,
2352         };
2353
2354         dst = ip6_route_output(net, NULL, &fl6);
2355         if (!dst->error)
2356                 __ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu));
2357         dst_release(dst);
2358 }
2359 EXPORT_SYMBOL_GPL(ip6_update_pmtu);
2360
2361 void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu)
2362 {
2363         struct dst_entry *dst;
2364
2365         ip6_update_pmtu(skb, sock_net(sk), mtu,
2366                         sk->sk_bound_dev_if, sk->sk_mark, sk->sk_uid);
2367
2368         dst = __sk_dst_get(sk);
2369         if (!dst || !dst->obsolete ||
2370             dst->ops->check(dst, inet6_sk(sk)->dst_cookie))
2371                 return;
2372
2373         bh_lock_sock(sk);
2374         if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(&sk->sk_v6_daddr))
2375                 ip6_datagram_dst_update(sk, false);
2376         bh_unlock_sock(sk);
2377 }
2378 EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu);
2379
2380 void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst,
2381                            const struct flowi6 *fl6)
2382 {
2383 #ifdef CONFIG_IPV6_SUBTREES
2384         struct ipv6_pinfo *np = inet6_sk(sk);
2385 #endif
2386
2387         ip6_dst_store(sk, dst,
2388                       ipv6_addr_equal(&fl6->daddr, &sk->sk_v6_daddr) ?
2389                       &sk->sk_v6_daddr : NULL,
2390 #ifdef CONFIG_IPV6_SUBTREES
2391                       ipv6_addr_equal(&fl6->saddr, &np->saddr) ?
2392                       &np->saddr :
2393 #endif
2394                       NULL);
2395 }
2396
2397 /* Handle redirects */
2398 struct ip6rd_flowi {
2399         struct flowi6 fl6;
2400         struct in6_addr gateway;
2401 };
2402
2403 static struct rt6_info *__ip6_route_redirect(struct net *net,
2404                                              struct fib6_table *table,
2405                                              struct flowi6 *fl6,
2406                                              const struct sk_buff *skb,
2407                                              int flags)
2408 {
2409         struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6;
2410         struct rt6_info *ret = NULL, *rt_cache;
2411         struct fib6_info *rt;
2412         struct fib6_node *fn;
2413
2414         /* Get the "current" route for this destination and
2415          * check if the redirect has come from appropriate router.
2416          *
2417          * RFC 4861 specifies that redirects should only be
2418          * accepted if they come from the nexthop to the target.
2419          * Due to the way the routes are chosen, this notion
2420          * is a bit fuzzy and one might need to check all possible
2421          * routes.
2422          */
2423
2424         rcu_read_lock();
2425         fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
2426 restart:
2427         for_each_fib6_node_rt_rcu(fn) {
2428                 if (rt->fib6_nh.nh_flags & RTNH_F_DEAD)
2429                         continue;
2430                 if (fib6_check_expired(rt))
2431                         continue;
2432                 if (rt->fib6_flags & RTF_REJECT)
2433                         break;
2434                 if (!(rt->fib6_flags & RTF_GATEWAY))
2435                         continue;
2436                 if (fl6->flowi6_oif != rt->fib6_nh.nh_dev->ifindex)
2437                         continue;
2438                 /* rt_cache's gateway might be different from its 'parent'
2439                  * in the case of an ip redirect.
2440                  * So we keep searching in the exception table if the gateway
2441                  * is different.
2442                  */
2443                 if (!ipv6_addr_equal(&rdfl->gateway, &rt->fib6_nh.nh_gw)) {
2444                         rt_cache = rt6_find_cached_rt(rt,
2445                                                       &fl6->daddr,
2446                                                       &fl6->saddr);
2447                         if (rt_cache &&
2448                             ipv6_addr_equal(&rdfl->gateway,
2449                                             &rt_cache->rt6i_gateway)) {
2450                                 ret = rt_cache;
2451                                 break;
2452                         }
2453                         continue;
2454                 }
2455                 break;
2456         }
2457
2458         if (!rt)
2459                 rt = net->ipv6.fib6_null_entry;
2460         else if (rt->fib6_flags & RTF_REJECT) {
2461                 ret = net->ipv6.ip6_null_entry;
2462                 goto out;
2463         }
2464
2465         if (rt == net->ipv6.fib6_null_entry) {
2466                 fn = fib6_backtrack(fn, &fl6->saddr);
2467                 if (fn)
2468                         goto restart;
2469         }
2470
2471 out:
2472         if (ret)
2473                 ip6_hold_safe(net, &ret, true);
2474         else
2475                 ret = ip6_create_rt_rcu(rt);
2476
2477         rcu_read_unlock();
2478
2479         trace_fib6_table_lookup(net, rt, table, fl6);
2480         return ret;
2481 };
2482
2483 static struct dst_entry *ip6_route_redirect(struct net *net,
2484                                             const struct flowi6 *fl6,
2485                                             const struct sk_buff *skb,
2486                                             const struct in6_addr *gateway)
2487 {
2488         int flags = RT6_LOOKUP_F_HAS_SADDR;
2489         struct ip6rd_flowi rdfl;
2490
2491         rdfl.fl6 = *fl6;
2492         rdfl.gateway = *gateway;
2493
2494         return fib6_rule_lookup(net, &rdfl.fl6, skb,
2495                                 flags, __ip6_route_redirect);
2496 }
2497
2498 void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark,
2499                   kuid_t uid)
2500 {
2501         const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
2502         struct dst_entry *dst;
2503         struct flowi6 fl6 = {
2504                 .flowi6_iif = LOOPBACK_IFINDEX,
2505                 .flowi6_oif = oif,
2506                 .flowi6_mark = mark,
2507                 .daddr = iph->daddr,
2508                 .saddr = iph->saddr,
2509                 .flowlabel = ip6_flowinfo(iph),
2510                 .flowi6_uid = uid,
2511         };
2512
2513         dst = ip6_route_redirect(net, &fl6, skb, &ipv6_hdr(skb)->saddr);
2514         rt6_do_redirect(dst, NULL, skb);
2515         dst_release(dst);
2516 }
2517 EXPORT_SYMBOL_GPL(ip6_redirect);
2518
2519 void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif)
2520 {
2521         const struct ipv6hdr *iph = ipv6_hdr(skb);
2522         const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb);
2523         struct dst_entry *dst;
2524         struct flowi6 fl6 = {
2525                 .flowi6_iif = LOOPBACK_IFINDEX,
2526                 .flowi6_oif = oif,
2527                 .daddr = msg->dest,
2528                 .saddr = iph->daddr,
2529                 .flowi6_uid = sock_net_uid(net, NULL),
2530         };
2531
2532         dst = ip6_route_redirect(net, &fl6, skb, &iph->saddr);
2533         rt6_do_redirect(dst, NULL, skb);
2534         dst_release(dst);
2535 }
2536
2537 void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk)
2538 {
2539         ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, sk->sk_mark,
2540                      sk->sk_uid);
2541 }
2542 EXPORT_SYMBOL_GPL(ip6_sk_redirect);
2543
2544 static unsigned int ip6_default_advmss(const struct dst_entry *dst)
2545 {
2546         struct net_device *dev = dst->dev;
2547         unsigned int mtu = dst_mtu(dst);
2548         struct net *net = dev_net(dev);
2549
2550         mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
2551
2552         if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss)
2553                 mtu = net->ipv6.sysctl.ip6_rt_min_advmss;
2554
2555         /*
2556          * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and
2557          * corresponding MSS is IPV6_MAXPLEN - tcp_header_size.
2558          * IPV6_MAXPLEN is also valid and means: "any MSS,
2559          * rely only on pmtu discovery"
2560          */
2561         if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr))
2562                 mtu = IPV6_MAXPLEN;
2563         return mtu;
2564 }
2565
2566 static unsigned int ip6_mtu(const struct dst_entry *dst)
2567 {
2568         struct inet6_dev *idev;
2569         unsigned int mtu;
2570
2571         mtu = dst_metric_raw(dst, RTAX_MTU);
2572         if (mtu)
2573                 goto out;
2574
2575         mtu = IPV6_MIN_MTU;
2576
2577         rcu_read_lock();
2578         idev = __in6_dev_get(dst->dev);
2579         if (idev)
2580                 mtu = idev->cnf.mtu6;
2581         rcu_read_unlock();
2582
2583 out:
2584         mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
2585
2586         return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
2587 }
2588
2589 /* MTU selection:
2590  * 1. mtu on route is locked - use it
2591  * 2. mtu from nexthop exception
2592  * 3. mtu from egress device
2593  *
2594  * based on ip6_dst_mtu_forward and exception logic of
2595  * rt6_find_cached_rt; called with rcu_read_lock
2596  */
2597 u32 ip6_mtu_from_fib6(struct fib6_info *f6i, struct in6_addr *daddr,
2598                       struct in6_addr *saddr)
2599 {
2600         struct rt6_exception_bucket *bucket;
2601         struct rt6_exception *rt6_ex;
2602         struct in6_addr *src_key;
2603         struct inet6_dev *idev;
2604         u32 mtu = 0;
2605
2606         if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) {
2607                 mtu = f6i->fib6_pmtu;
2608                 if (mtu)
2609                         goto out;
2610         }
2611
2612         src_key = NULL;
2613 #ifdef CONFIG_IPV6_SUBTREES
2614         if (f6i->fib6_src.plen)
2615                 src_key = saddr;
2616 #endif
2617
2618         bucket = rcu_dereference(f6i->rt6i_exception_bucket);
2619         rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key);
2620         if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i))
2621                 mtu = dst_metric_raw(&rt6_ex->rt6i->dst, RTAX_MTU);
2622
2623         if (likely(!mtu)) {
2624                 struct net_device *dev = fib6_info_nh_dev(f6i);
2625
2626                 mtu = IPV6_MIN_MTU;
2627                 idev = __in6_dev_get(dev);
2628                 if (idev && idev->cnf.mtu6 > mtu)
2629                         mtu = idev->cnf.mtu6;
2630         }
2631
2632         mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
2633 out:
2634         return mtu - lwtunnel_headroom(fib6_info_nh_lwt(f6i), mtu);
2635 }
2636
2637 struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
2638                                   struct flowi6 *fl6)
2639 {
2640         struct dst_entry *dst;
2641         struct rt6_info *rt;
2642         struct inet6_dev *idev = in6_dev_get(dev);
2643         struct net *net = dev_net(dev);
2644
2645         if (unlikely(!idev))
2646                 return ERR_PTR(-ENODEV);
2647
2648         rt = ip6_dst_alloc(net, dev, 0);
2649         if (unlikely(!rt)) {
2650                 in6_dev_put(idev);
2651                 dst = ERR_PTR(-ENOMEM);
2652                 goto out;
2653         }
2654
2655         rt->dst.flags |= DST_HOST;
2656         rt->dst.input = ip6_input;
2657         rt->dst.output  = ip6_output;
2658         rt->rt6i_gateway  = fl6->daddr;
2659         rt->rt6i_dst.addr = fl6->daddr;
2660         rt->rt6i_dst.plen = 128;
2661         rt->rt6i_idev     = idev;
2662         dst_metric_set(&rt->dst, RTAX_HOPLIMIT, 0);
2663
2664         /* Add this dst into uncached_list so that rt6_disable_ip() can
2665          * do proper release of the net_device
2666          */
2667         rt6_uncached_list_add(rt);
2668         atomic_inc(&net->ipv6.rt6_stats->fib_rt_uncache);
2669
2670         dst = xfrm_lookup(net, &rt->dst, flowi6_to_flowi(fl6), NULL, 0);
2671
2672 out:
2673         return dst;
2674 }
2675
2676 static int ip6_dst_gc(struct dst_ops *ops)
2677 {
2678         struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
2679         int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
2680         int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;
2681         int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
2682         int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
2683         unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
2684         int entries;
2685
2686         entries = dst_entries_get_fast(ops);
2687         if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
2688             entries <= rt_max_size)
2689                 goto out;
2690
2691         net->ipv6.ip6_rt_gc_expire++;
2692         fib6_run_gc(net->ipv6.ip6_rt_gc_expire, net, true);
2693         entries = dst_entries_get_slow(ops);
2694         if (entries < ops->gc_thresh)
2695                 net->ipv6.ip6_rt_gc_expire = rt_gc_timeout>>1;
2696 out:
2697         net->ipv6.ip6_rt_gc_expire -= net->ipv6.ip6_rt_gc_expire>>rt_elasticity;
2698         return entries > rt_max_size;
2699 }
2700
2701 static struct rt6_info *ip6_nh_lookup_table(struct net *net,
2702                                             struct fib6_config *cfg,
2703                                             const struct in6_addr *gw_addr,
2704                                             u32 tbid, int flags)
2705 {
2706         struct flowi6 fl6 = {
2707                 .flowi6_oif = cfg->fc_ifindex,
2708                 .daddr = *gw_addr,
2709                 .saddr = cfg->fc_prefsrc,
2710         };
2711         struct fib6_table *table;
2712         struct rt6_info *rt;
2713
2714         table = fib6_get_table(net, tbid);
2715         if (!table)
2716                 return NULL;
2717
2718         if (!ipv6_addr_any(&cfg->fc_prefsrc))
2719                 flags |= RT6_LOOKUP_F_HAS_SADDR;
2720
2721         flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE;
2722         rt = ip6_pol_route(net, table, cfg->fc_ifindex, &fl6, NULL, flags);
2723
2724         /* if table lookup failed, fall back to full lookup */
2725         if (rt == net->ipv6.ip6_null_entry) {
2726                 ip6_rt_put(rt);
2727                 rt = NULL;
2728         }
2729
2730         return rt;
2731 }
2732
2733 static int ip6_route_check_nh_onlink(struct net *net,
2734                                      struct fib6_config *cfg,
2735                                      const struct net_device *dev,
2736                                      struct netlink_ext_ack *extack)
2737 {
2738         u32 tbid = l3mdev_fib_table(dev) ? : RT_TABLE_MAIN;
2739         const struct in6_addr *gw_addr = &cfg->fc_gateway;
2740         u32 flags = RTF_LOCAL | RTF_ANYCAST | RTF_REJECT;
2741         struct rt6_info *grt;
2742         int err;
2743
2744         err = 0;
2745         grt = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, 0);
2746         if (grt) {
2747                 if (!grt->dst.error &&
2748                     (grt->rt6i_flags & flags || dev != grt->dst.dev)) {
2749                         NL_SET_ERR_MSG(extack,
2750                                        "Nexthop has invalid gateway or device mismatch");
2751                         err = -EINVAL;
2752                 }
2753
2754                 ip6_rt_put(grt);
2755         }
2756
2757         return err;
2758 }
2759
2760 static int ip6_route_check_nh(struct net *net,
2761                               struct fib6_config *cfg,
2762                               struct net_device **_dev,
2763                               struct inet6_dev **idev)
2764 {
2765         const struct in6_addr *gw_addr = &cfg->fc_gateway;
2766         struct net_device *dev = _dev ? *_dev : NULL;
2767         struct rt6_info *grt = NULL;
2768         int err = -EHOSTUNREACH;
2769
2770         if (cfg->fc_table) {
2771                 int flags = RT6_LOOKUP_F_IFACE;
2772
2773                 grt = ip6_nh_lookup_table(net, cfg, gw_addr,
2774                                           cfg->fc_table, flags);
2775                 if (grt) {
2776                         if (grt->rt6i_flags & RTF_GATEWAY ||
2777                             (dev && dev != grt->dst.dev)) {
2778                                 ip6_rt_put(grt);
2779                                 grt = NULL;
2780                         }
2781                 }
2782         }
2783
2784         if (!grt)
2785                 grt = rt6_lookup(net, gw_addr, NULL, cfg->fc_ifindex, NULL, 1);
2786
2787         if (!grt)
2788                 goto out;
2789
2790         if (dev) {
2791                 if (dev != grt->dst.dev) {
2792                         ip6_rt_put(grt);
2793                         goto out;
2794                 }
2795         } else {
2796                 *_dev = dev = grt->dst.dev;
2797                 *idev = grt->rt6i_idev;
2798                 dev_hold(dev);
2799                 in6_dev_hold(grt->rt6i_idev);
2800         }
2801
2802         if (!(grt->rt6i_flags & RTF_GATEWAY))
2803                 err = 0;
2804
2805         ip6_rt_put(grt);
2806
2807 out:
2808         return err;
2809 }
2810
2811 static int ip6_validate_gw(struct net *net, struct fib6_config *cfg,
2812                            struct net_device **_dev, struct inet6_dev **idev,
2813                            struct netlink_ext_ack *extack)
2814 {
2815         const struct in6_addr *gw_addr = &cfg->fc_gateway;
2816         int gwa_type = ipv6_addr_type(gw_addr);
2817         bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true;
2818         const struct net_device *dev = *_dev;
2819         bool need_addr_check = !dev;
2820         int err = -EINVAL;
2821
2822         /* if gw_addr is local we will fail to detect this in case
2823          * address is still TENTATIVE (DAD in progress). rt6_lookup()
2824          * will return already-added prefix route via interface that
2825          * prefix route was assigned to, which might be non-loopback.
2826          */
2827         if (dev &&
2828             ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) {
2829                 NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
2830                 goto out;
2831         }
2832
2833         if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) {
2834                 /* IPv6 strictly inhibits using not link-local
2835                  * addresses as nexthop address.
2836                  * Otherwise, router will not able to send redirects.
2837                  * It is very good, but in some (rare!) circumstances
2838                  * (SIT, PtP, NBMA NOARP links) it is handy to allow
2839                  * some exceptions. --ANK
2840                  * We allow IPv4-mapped nexthops to support RFC4798-type
2841                  * addressing
2842                  */
2843                 if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) {
2844                         NL_SET_ERR_MSG(extack, "Invalid gateway address");
2845                         goto out;
2846                 }
2847
2848                 if (cfg->fc_flags & RTNH_F_ONLINK)
2849                         err = ip6_route_check_nh_onlink(net, cfg, dev, extack);
2850                 else
2851                         err = ip6_route_check_nh(net, cfg, _dev, idev);
2852
2853                 if (err)
2854                         goto out;
2855         }
2856
2857         /* reload in case device was changed */
2858         dev = *_dev;
2859
2860         err = -EINVAL;
2861         if (!dev) {
2862                 NL_SET_ERR_MSG(extack, "Egress device not specified");
2863                 goto out;
2864         } else if (dev->flags & IFF_LOOPBACK) {
2865                 NL_SET_ERR_MSG(extack,
2866                                "Egress device can not be loopback device for this route");
2867                 goto out;
2868         }
2869
2870         /* if we did not check gw_addr above, do so now that the
2871          * egress device has been resolved.
2872          */
2873         if (need_addr_check &&
2874             ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) {
2875                 NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
2876                 goto out;
2877         }
2878
2879         err = 0;
2880 out:
2881         return err;
2882 }
2883
2884 static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg,
2885                                               gfp_t gfp_flags,
2886                                               struct netlink_ext_ack *extack)
2887 {
2888         struct net *net = cfg->fc_nlinfo.nl_net;
2889         struct fib6_info *rt = NULL;
2890         struct net_device *dev = NULL;
2891         struct inet6_dev *idev = NULL;
2892         struct fib6_table *table;
2893         int addr_type;
2894         int err = -EINVAL;
2895
2896         /* RTF_PCPU is an internal flag; can not be set by userspace */
2897         if (cfg->fc_flags & RTF_PCPU) {
2898                 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU");
2899                 goto out;
2900         }
2901
2902         /* RTF_CACHE is an internal flag; can not be set by userspace */
2903         if (cfg->fc_flags & RTF_CACHE) {
2904                 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE");
2905                 goto out;
2906         }
2907
2908         if (cfg->fc_type > RTN_MAX) {
2909                 NL_SET_ERR_MSG(extack, "Invalid route type");
2910                 goto out;
2911         }
2912
2913         if (cfg->fc_dst_len > 128) {
2914                 NL_SET_ERR_MSG(extack, "Invalid prefix length");
2915                 goto out;
2916         }
2917         if (cfg->fc_src_len > 128) {
2918                 NL_SET_ERR_MSG(extack, "Invalid source address length");
2919                 goto out;
2920         }
2921 #ifndef CONFIG_IPV6_SUBTREES
2922         if (cfg->fc_src_len) {
2923                 NL_SET_ERR_MSG(extack,
2924                                "Specifying source address requires IPV6_SUBTREES to be enabled");
2925                 goto out;
2926         }
2927 #endif
2928         if (cfg->fc_ifindex) {
2929                 err = -ENODEV;
2930                 dev = dev_get_by_index(net, cfg->fc_ifindex);
2931                 if (!dev)
2932                         goto out;
2933                 idev = in6_dev_get(dev);
2934                 if (!idev)
2935                         goto out;
2936         }
2937
2938         if (cfg->fc_metric == 0)
2939                 cfg->fc_metric = IP6_RT_PRIO_USER;
2940
2941         if (cfg->fc_flags & RTNH_F_ONLINK) {
2942                 if (!dev) {
2943                         NL_SET_ERR_MSG(extack,
2944                                        "Nexthop device required for onlink");
2945                         err = -ENODEV;
2946                         goto out;
2947                 }
2948
2949                 if (!(dev->flags & IFF_UP)) {
2950                         NL_SET_ERR_MSG(extack, "Nexthop device is not up");
2951                         err = -ENETDOWN;
2952                         goto out;
2953                 }
2954         }
2955
2956         err = -ENOBUFS;
2957         if (cfg->fc_nlinfo.nlh &&
2958             !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) {
2959                 table = fib6_get_table(net, cfg->fc_table);
2960                 if (!table) {
2961                         pr_warn("NLM_F_CREATE should be specified when creating new route\n");
2962                         table = fib6_new_table(net, cfg->fc_table);
2963                 }
2964         } else {
2965                 table = fib6_new_table(net, cfg->fc_table);
2966         }
2967
2968         if (!table)
2969                 goto out;
2970
2971         err = -ENOMEM;
2972         rt = fib6_info_alloc(gfp_flags);
2973         if (!rt)
2974                 goto out;
2975
2976         rt->fib6_metrics = ip_fib_metrics_init(net, cfg->fc_mx, cfg->fc_mx_len);
2977         if (IS_ERR(rt->fib6_metrics)) {
2978                 err = PTR_ERR(rt->fib6_metrics);
2979                 goto out;
2980         }
2981
2982         if (cfg->fc_flags & RTF_ADDRCONF)
2983                 rt->dst_nocount = true;
2984
2985         if (cfg->fc_flags & RTF_EXPIRES)
2986                 fib6_set_expires(rt, jiffies +
2987                                 clock_t_to_jiffies(cfg->fc_expires));
2988         else
2989                 fib6_clean_expires(rt);
2990
2991         if (cfg->fc_protocol == RTPROT_UNSPEC)
2992                 cfg->fc_protocol = RTPROT_BOOT;
2993         rt->fib6_protocol = cfg->fc_protocol;
2994
2995         addr_type = ipv6_addr_type(&cfg->fc_dst);
2996
2997         if (cfg->fc_encap) {
2998                 struct lwtunnel_state *lwtstate;
2999
3000                 err = lwtunnel_build_state(cfg->fc_encap_type,
3001                                            cfg->fc_encap, AF_INET6, cfg,
3002                                            &lwtstate, extack);
3003                 if (err)
3004                         goto out;
3005                 rt->fib6_nh.nh_lwtstate = lwtstate_get(lwtstate);
3006         }
3007
3008         ipv6_addr_prefix(&rt->fib6_dst.addr, &cfg->fc_dst, cfg->fc_dst_len);
3009         rt->fib6_dst.plen = cfg->fc_dst_len;
3010         if (rt->fib6_dst.plen == 128)
3011                 rt->dst_host = true;
3012
3013 #ifdef CONFIG_IPV6_SUBTREES
3014         ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len);
3015         rt->fib6_src.plen = cfg->fc_src_len;
3016 #endif
3017
3018         rt->fib6_metric = cfg->fc_metric;
3019         rt->fib6_nh.nh_weight = 1;
3020
3021         rt->fib6_type = cfg->fc_type;
3022
3023         /* We cannot add true routes via loopback here,
3024            they would result in kernel looping; promote them to reject routes
3025          */
3026         if ((cfg->fc_flags & RTF_REJECT) ||
3027             (dev && (dev->flags & IFF_LOOPBACK) &&
3028              !(addr_type & IPV6_ADDR_LOOPBACK) &&
3029              !(cfg->fc_flags & RTF_LOCAL))) {
3030                 /* hold loopback dev/idev if we haven't done so. */
3031                 if (dev != net->loopback_dev) {
3032                         if (dev) {
3033                                 dev_put(dev);
3034                                 in6_dev_put(idev);
3035                         }
3036                         dev = net->loopback_dev;
3037                         dev_hold(dev);
3038                         idev = in6_dev_get(dev);
3039                         if (!idev) {
3040                                 err = -ENODEV;
3041                                 goto out;
3042                         }
3043                 }
3044                 rt->fib6_flags = RTF_REJECT|RTF_NONEXTHOP;
3045                 goto install_route;
3046         }
3047
3048         if (cfg->fc_flags & RTF_GATEWAY) {
3049                 err = ip6_validate_gw(net, cfg, &dev, &idev, extack);
3050                 if (err)
3051                         goto out;
3052
3053                 rt->fib6_nh.nh_gw = cfg->fc_gateway;
3054         }
3055
3056         err = -ENODEV;
3057         if (!dev)
3058                 goto out;
3059
3060         if (idev->cnf.disable_ipv6) {
3061                 NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device");
3062                 err = -EACCES;
3063                 goto out;
3064         }
3065
3066         if (!(dev->flags & IFF_UP)) {
3067                 NL_SET_ERR_MSG(extack, "Nexthop device is not up");
3068                 err = -ENETDOWN;
3069                 goto out;
3070         }
3071
3072         if (!ipv6_addr_any(&cfg->fc_prefsrc)) {
3073                 if (!ipv6_chk_addr(net, &cfg->fc_prefsrc, dev, 0)) {
3074                         NL_SET_ERR_MSG(extack, "Invalid source address");
3075                         err = -EINVAL;
3076                         goto out;
3077                 }
3078                 rt->fib6_prefsrc.addr = cfg->fc_prefsrc;
3079                 rt->fib6_prefsrc.plen = 128;
3080         } else
3081                 rt->fib6_prefsrc.plen = 0;
3082
3083         rt->fib6_flags = cfg->fc_flags;
3084
3085 install_route:
3086         if (!(rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) &&
3087             !netif_carrier_ok(dev))
3088                 rt->fib6_nh.nh_flags |= RTNH_F_LINKDOWN;
3089         rt->fib6_nh.nh_flags |= (cfg->fc_flags & RTNH_F_ONLINK);
3090         rt->fib6_nh.nh_dev = dev;
3091         rt->fib6_table = table;
3092
3093         if (idev)
3094                 in6_dev_put(idev);
3095
3096         return rt;
3097 out:
3098         if (dev)
3099                 dev_put(dev);
3100         if (idev)
3101                 in6_dev_put(idev);
3102
3103         fib6_info_release(rt);
3104         return ERR_PTR(err);
3105 }
3106
3107 int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags,
3108                   struct netlink_ext_ack *extack)
3109 {
3110         struct fib6_info *rt;
3111         int err;
3112
3113         rt = ip6_route_info_create(cfg, gfp_flags, extack);
3114         if (IS_ERR(rt))
3115                 return PTR_ERR(rt);
3116
3117         err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack);
3118         fib6_info_release(rt);
3119
3120         return err;
3121 }
3122
3123 static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info)
3124 {
3125         struct net *net = info->nl_net;
3126         struct fib6_table *table;
3127         int err;
3128
3129         if (rt == net->ipv6.fib6_null_entry) {
3130                 err = -ENOENT;
3131                 goto out;
3132         }
3133
3134         table = rt->fib6_table;
3135         spin_lock_bh(&table->tb6_lock);
3136         err = fib6_del(rt, info);
3137         spin_unlock_bh(&table->tb6_lock);
3138
3139 out:
3140         fib6_info_release(rt);
3141         return err;
3142 }
3143
3144 int ip6_del_rt(struct net *net, struct fib6_info *rt)
3145 {
3146         struct nl_info info = { .nl_net = net };
3147
3148         return __ip6_del_rt(rt, &info);
3149 }
3150
3151 static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg)
3152 {
3153         struct nl_info *info = &cfg->fc_nlinfo;
3154         struct net *net = info->nl_net;
3155         struct sk_buff *skb = NULL;
3156         struct fib6_table *table;
3157         int err = -ENOENT;
3158
3159         if (rt == net->ipv6.fib6_null_entry)
3160                 goto out_put;
3161         table = rt->fib6_table;
3162         spin_lock_bh(&table->tb6_lock);
3163
3164         if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) {
3165                 struct fib6_info *sibling, *next_sibling;
3166
3167                 /* prefer to send a single notification with all hops */
3168                 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
3169                 if (skb) {
3170                         u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
3171
3172                         if (rt6_fill_node(net, skb, rt, NULL,
3173                                           NULL, NULL, 0, RTM_DELROUTE,
3174                                           info->portid, seq, 0) < 0) {
3175                                 kfree_skb(skb);
3176                                 skb = NULL;
3177                         } else
3178                                 info->skip_notify = 1;
3179                 }
3180
3181                 list_for_each_entry_safe(sibling, next_sibling,
3182                                          &rt->fib6_siblings,
3183                                          fib6_siblings) {
3184                         err = fib6_del(sibling, info);
3185                         if (err)
3186                                 goto out_unlock;
3187                 }
3188         }
3189
3190         err = fib6_del(rt, info);
3191 out_unlock:
3192         spin_unlock_bh(&table->tb6_lock);
3193 out_put:
3194         fib6_info_release(rt);
3195
3196         if (skb) {
3197                 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
3198                             info->nlh, gfp_any());
3199         }
3200         return err;
3201 }
3202
3203 static int ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg)
3204 {
3205         int rc = -ESRCH;
3206
3207         if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex)
3208                 goto out;
3209
3210         if (cfg->fc_flags & RTF_GATEWAY &&
3211             !ipv6_addr_equal(&cfg->fc_gateway, &rt->rt6i_gateway))
3212                 goto out;
3213         if (dst_hold_safe(&rt->dst))
3214                 rc = rt6_remove_exception_rt(rt);
3215 out:
3216         return rc;
3217 }
3218
3219 static int ip6_route_del(struct fib6_config *cfg,
3220                          struct netlink_ext_ack *extack)
3221 {
3222         struct rt6_info *rt_cache;
3223         struct fib6_table *table;
3224         struct fib6_info *rt;
3225         struct fib6_node *fn;
3226         int err = -ESRCH;
3227
3228         table = fib6_get_table(cfg->fc_nlinfo.nl_net, cfg->fc_table);
3229         if (!table) {
3230                 NL_SET_ERR_MSG(extack, "FIB table does not exist");
3231                 return err;
3232         }
3233
3234         rcu_read_lock();
3235
3236         fn = fib6_locate(&table->tb6_root,
3237                          &cfg->fc_dst, cfg->fc_dst_len,
3238                          &cfg->fc_src, cfg->fc_src_len,
3239                          !(cfg->fc_flags & RTF_CACHE));
3240
3241         if (fn) {
3242                 for_each_fib6_node_rt_rcu(fn) {
3243                         if (cfg->fc_flags & RTF_CACHE) {
3244                                 int rc;
3245
3246                                 rt_cache = rt6_find_cached_rt(rt, &cfg->fc_dst,
3247                                                               &cfg->fc_src);
3248                                 if (rt_cache) {
3249                                         rc = ip6_del_cached_rt(rt_cache, cfg);
3250                                         if (rc != -ESRCH) {
3251                                                 rcu_read_unlock();
3252                                                 return rc;
3253                                         }
3254                                 }
3255                                 continue;
3256                         }
3257                         if (cfg->fc_ifindex &&
3258                             (!rt->fib6_nh.nh_dev ||
3259                              rt->fib6_nh.nh_dev->ifindex != cfg->fc_ifindex))
3260                                 continue;
3261                         if (cfg->fc_flags & RTF_GATEWAY &&
3262                             !ipv6_addr_equal(&cfg->fc_gateway, &rt->fib6_nh.nh_gw))
3263                                 continue;
3264                         if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric)
3265                                 continue;
3266                         if (cfg->fc_protocol && cfg->fc_protocol != rt->fib6_protocol)
3267                                 continue;
3268                         if (!fib6_info_hold_safe(rt))
3269                                 continue;
3270                         rcu_read_unlock();
3271
3272                         /* if gateway was specified only delete the one hop */
3273                         if (cfg->fc_flags & RTF_GATEWAY)
3274                                 return __ip6_del_rt(rt, &cfg->fc_nlinfo);
3275
3276                         return __ip6_del_rt_siblings(rt, cfg);
3277                 }
3278         }
3279         rcu_read_unlock();
3280
3281         return err;
3282 }
3283
3284 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
3285 {
3286         struct netevent_redirect netevent;
3287         struct rt6_info *rt, *nrt = NULL;
3288         struct ndisc_options ndopts;
3289         struct inet6_dev *in6_dev;
3290         struct neighbour *neigh;
3291         struct fib6_info *from;
3292         struct rd_msg *msg;
3293         int optlen, on_link;
3294         u8 *lladdr;
3295
3296         optlen = skb_tail_pointer(skb) - skb_transport_header(skb);
3297         optlen -= sizeof(*msg);
3298
3299         if (optlen < 0) {
3300                 net_dbg_ratelimited("rt6_do_redirect: packet too short\n");
3301                 return;
3302         }
3303
3304         msg = (struct rd_msg *)icmp6_hdr(skb);
3305
3306         if (ipv6_addr_is_multicast(&msg->dest)) {
3307                 net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n");
3308                 return;
3309         }
3310
3311         on_link = 0;
3312         if (ipv6_addr_equal(&msg->dest, &msg->target)) {
3313                 on_link = 1;
3314         } else if (ipv6_addr_type(&msg->target) !=
3315                    (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) {
3316                 net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n");
3317                 return;
3318         }
3319
3320         in6_dev = __in6_dev_get(skb->dev);
3321         if (!in6_dev)
3322                 return;
3323         if (in6_dev->cnf.forwarding || !in6_dev->cnf.accept_redirects)
3324                 return;
3325
3326         /* RFC2461 8.1:
3327          *      The IP source address of the Redirect MUST be the same as the current
3328          *      first-hop router for the specified ICMP Destination Address.
3329          */
3330
3331         if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) {
3332                 net_dbg_ratelimited("rt6_redirect: invalid ND options\n");
3333                 return;
3334         }
3335
3336         lladdr = NULL;
3337         if (ndopts.nd_opts_tgt_lladdr) {
3338                 lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr,
3339                                              skb->dev);
3340                 if (!lladdr) {
3341                         net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n");
3342                         return;
3343                 }
3344         }
3345
3346         rt = (struct rt6_info *) dst;
3347         if (rt->rt6i_flags & RTF_REJECT) {
3348                 net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n");
3349                 return;
3350         }
3351
3352         /* Redirect received -> path was valid.
3353          * Look, redirects are sent only in response to data packets,
3354          * so that this nexthop apparently is reachable. --ANK
3355          */
3356         dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr);
3357
3358         neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1);
3359         if (!neigh)
3360                 return;
3361
3362         /*
3363          *      We have finally decided to accept it.
3364          */
3365
3366         ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
3367                      NEIGH_UPDATE_F_WEAK_OVERRIDE|
3368                      NEIGH_UPDATE_F_OVERRIDE|
3369                      (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
3370                                      NEIGH_UPDATE_F_ISROUTER)),
3371                      NDISC_REDIRECT, &ndopts);
3372
3373         rcu_read_lock();
3374         from = rcu_dereference(rt->from);
3375         /* This fib6_info_hold() is safe here because we hold reference to rt
3376          * and rt already holds reference to fib6_info.
3377          */
3378         fib6_info_hold(from);
3379         rcu_read_unlock();
3380
3381         nrt = ip6_rt_cache_alloc(from, &msg->dest, NULL);
3382         if (!nrt)
3383                 goto out;
3384
3385         nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE;
3386         if (on_link)
3387                 nrt->rt6i_flags &= ~RTF_GATEWAY;
3388
3389         nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key;
3390
3391         /* No need to remove rt from the exception table if rt is
3392          * a cached route because rt6_insert_exception() will
3393          * takes care of it
3394          */
3395         if (rt6_insert_exception(nrt, from)) {
3396                 dst_release_immediate(&nrt->dst);
3397                 goto out;
3398         }
3399
3400         netevent.old = &rt->dst;
3401         netevent.new = &nrt->dst;
3402         netevent.daddr = &msg->dest;
3403         netevent.neigh = neigh;
3404         call_netevent_notifiers(NETEVENT_REDIRECT, &netevent);
3405
3406 out:
3407         fib6_info_release(from);
3408         neigh_release(neigh);
3409 }
3410
3411 #ifdef CONFIG_IPV6_ROUTE_INFO
3412 static struct fib6_info *rt6_get_route_info(struct net *net,
3413                                            const struct in6_addr *prefix, int prefixlen,
3414                                            const struct in6_addr *gwaddr,
3415                                            struct net_device *dev)
3416 {
3417         u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
3418         int ifindex = dev->ifindex;
3419         struct fib6_node *fn;
3420         struct fib6_info *rt = NULL;
3421         struct fib6_table *table;
3422
3423         table = fib6_get_table(net, tb_id);
3424         if (!table)
3425                 return NULL;
3426
3427         rcu_read_lock();
3428         fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true);
3429         if (!fn)
3430                 goto out;
3431
3432         for_each_fib6_node_rt_rcu(fn) {
3433                 if (rt->fib6_nh.nh_dev->ifindex != ifindex)
3434                         continue;
3435                 if ((rt->fib6_flags & (RTF_ROUTEINFO|RTF_GATEWAY)) != (RTF_ROUTEINFO|RTF_GATEWAY))
3436                         continue;
3437                 if (!ipv6_addr_equal(&rt->fib6_nh.nh_gw, gwaddr))
3438                         continue;
3439                 if (!fib6_info_hold_safe(rt))
3440                         continue;
3441                 break;
3442         }
3443 out:
3444         rcu_read_unlock();
3445         return rt;
3446 }
3447
3448 static struct fib6_info *rt6_add_route_info(struct net *net,
3449                                            const struct in6_addr *prefix, int prefixlen,
3450                                            const struct in6_addr *gwaddr,
3451                                            struct net_device *dev,
3452                                            unsigned int pref)
3453 {
3454         struct fib6_config cfg = {
3455                 .fc_metric      = IP6_RT_PRIO_USER,
3456                 .fc_ifindex     = dev->ifindex,
3457                 .fc_dst_len     = prefixlen,
3458                 .fc_flags       = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO |
3459                                   RTF_UP | RTF_PREF(pref),
3460                 .fc_protocol = RTPROT_RA,
3461                 .fc_type = RTN_UNICAST,
3462                 .fc_nlinfo.portid = 0,
3463                 .fc_nlinfo.nlh = NULL,
3464                 .fc_nlinfo.nl_net = net,
3465         };
3466
3467         cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO,
3468         cfg.fc_dst = *prefix;
3469         cfg.fc_gateway = *gwaddr;
3470
3471         /* We should treat it as a default route if prefix length is 0. */
3472         if (!prefixlen)
3473                 cfg.fc_flags |= RTF_DEFAULT;
3474
3475         ip6_route_add(&cfg, GFP_ATOMIC, NULL);
3476
3477         return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev);
3478 }
3479 #endif
3480
3481 struct fib6_info *rt6_get_dflt_router(struct net *net,
3482                                      const struct in6_addr *addr,
3483                                      struct net_device *dev)
3484 {
3485         u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT;
3486         struct fib6_info *rt;
3487         struct fib6_table *table;
3488
3489         table = fib6_get_table(net, tb_id);
3490         if (!table)
3491                 return NULL;
3492
3493         rcu_read_lock();
3494         for_each_fib6_node_rt_rcu(&table->tb6_root) {
3495                 if (dev == rt->fib6_nh.nh_dev &&
3496                     ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) &&
3497                     ipv6_addr_equal(&rt->fib6_nh.nh_gw, addr))
3498                         break;
3499         }
3500         if (rt && !fib6_info_hold_safe(rt))
3501                 rt = NULL;
3502         rcu_read_unlock();
3503         return rt;
3504 }
3505
3506 struct fib6_info *rt6_add_dflt_router(struct net *net,
3507                                      const struct in6_addr *gwaddr,
3508                                      struct net_device *dev,
3509                                      unsigned int pref)
3510 {
3511         struct fib6_config cfg = {
3512                 .fc_table       = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT,
3513                 .fc_metric      = IP6_RT_PRIO_USER,
3514                 .fc_ifindex     = dev->ifindex,
3515                 .fc_flags       = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT |
3516                                   RTF_UP | RTF_EXPIRES | RTF_PREF(pref),
3517                 .fc_protocol = RTPROT_RA,
3518                 .fc_type = RTN_UNICAST,
3519                 .fc_nlinfo.portid = 0,
3520                 .fc_nlinfo.nlh = NULL,
3521                 .fc_nlinfo.nl_net = net,
3522         };
3523
3524         cfg.fc_gateway = *gwaddr;
3525
3526         if (!ip6_route_add(&cfg, GFP_ATOMIC, NULL)) {
3527                 struct fib6_table *table;
3528
3529                 table = fib6_get_table(dev_net(dev), cfg.fc_table);
3530                 if (table)
3531                         table->flags |= RT6_TABLE_HAS_DFLT_ROUTER;
3532         }
3533
3534         return rt6_get_dflt_router(net, gwaddr, dev);
3535 }
3536
3537 static void __rt6_purge_dflt_routers(struct net *net,
3538                                      struct fib6_table *table)
3539 {
3540         struct fib6_info *rt;
3541
3542 restart:
3543         rcu_read_lock();
3544         for_each_fib6_node_rt_rcu(&table->tb6_root) {
3545                 struct net_device *dev = fib6_info_nh_dev(rt);
3546                 struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL;
3547
3548                 if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) &&
3549                     (!idev || idev->cnf.accept_ra != 2) &&
3550                     fib6_info_hold_safe(rt)) {
3551                         rcu_read_unlock();
3552                         ip6_del_rt(net, rt);
3553                         goto restart;
3554                 }
3555         }
3556         rcu_read_unlock();
3557
3558         table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER;
3559 }
3560
3561 void rt6_purge_dflt_routers(struct net *net)
3562 {
3563         struct fib6_table *table;
3564         struct hlist_head *head;
3565         unsigned int h;
3566
3567         rcu_read_lock();
3568
3569         for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
3570                 head = &net->ipv6.fib_table_hash[h];
3571                 hlist_for_each_entry_rcu(table, head, tb6_hlist) {
3572                         if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER)
3573                                 __rt6_purge_dflt_routers(net, table);
3574                 }
3575         }
3576
3577         rcu_read_unlock();
3578 }
3579
3580 static void rtmsg_to_fib6_config(struct net *net,
3581                                  struct in6_rtmsg *rtmsg,
3582                                  struct fib6_config *cfg)
3583 {
3584         *cfg = (struct fib6_config){
3585                 .fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ?
3586                          : RT6_TABLE_MAIN,
3587                 .fc_ifindex = rtmsg->rtmsg_ifindex,
3588                 .fc_metric = rtmsg->rtmsg_metric,
3589                 .fc_expires = rtmsg->rtmsg_info,
3590                 .fc_dst_len = rtmsg->rtmsg_dst_len,
3591                 .fc_src_len = rtmsg->rtmsg_src_len,
3592                 .fc_flags = rtmsg->rtmsg_flags,
3593                 .fc_type = rtmsg->rtmsg_type,
3594
3595                 .fc_nlinfo.nl_net = net,
3596
3597                 .fc_dst = rtmsg->rtmsg_dst,
3598                 .fc_src = rtmsg->rtmsg_src,
3599                 .fc_gateway = rtmsg->rtmsg_gateway,
3600         };
3601 }
3602
3603 int ipv6_route_ioctl(struct net *net, unsigned int cmd, void __user *arg)
3604 {
3605         struct fib6_config cfg;
3606         struct in6_rtmsg rtmsg;
3607         int err;
3608
3609         switch (cmd) {
3610         case SIOCADDRT:         /* Add a route */
3611         case SIOCDELRT:         /* Delete a route */
3612                 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
3613                         return -EPERM;
3614                 err = copy_from_user(&rtmsg, arg,
3615                                      sizeof(struct in6_rtmsg));
3616                 if (err)
3617                         return -EFAULT;
3618
3619                 rtmsg_to_fib6_config(net, &rtmsg, &cfg);
3620
3621                 rtnl_lock();
3622                 switch (cmd) {
3623                 case SIOCADDRT:
3624                         err = ip6_route_add(&cfg, GFP_KERNEL, NULL);
3625                         break;
3626                 case SIOCDELRT:
3627                         err = ip6_route_del(&cfg, NULL);
3628                         break;
3629                 default:
3630                         err = -EINVAL;
3631                 }
3632                 rtnl_unlock();
3633
3634                 return err;
3635         }
3636
3637         return -EINVAL;
3638 }
3639
3640 /*
3641  *      Drop the packet on the floor
3642  */
3643
3644 static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes)
3645 {
3646         int type;
3647         struct dst_entry *dst = skb_dst(skb);
3648         switch (ipstats_mib_noroutes) {
3649         case IPSTATS_MIB_INNOROUTES:
3650                 type = ipv6_addr_type(&ipv6_hdr(skb)->daddr);
3651                 if (type == IPV6_ADDR_ANY) {
3652                         IP6_INC_STATS(dev_net(dst->dev),
3653                                       __in6_dev_get_safely(skb->dev),
3654                                       IPSTATS_MIB_INADDRERRORS);
3655                         break;
3656                 }
3657                 /* FALLTHROUGH */
3658         case IPSTATS_MIB_OUTNOROUTES:
3659                 IP6_INC_STATS(dev_net(dst->dev), ip6_dst_idev(dst),
3660                               ipstats_mib_noroutes);
3661                 break;
3662         }
3663         icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, 0);
3664         kfree_skb(skb);
3665         return 0;
3666 }
3667
3668 static int ip6_pkt_discard(struct sk_buff *skb)
3669 {
3670         return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_INNOROUTES);
3671 }
3672
3673 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb)
3674 {
3675         skb->dev = skb_dst(skb)->dev;
3676         return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_OUTNOROUTES);
3677 }
3678
3679 static int ip6_pkt_prohibit(struct sk_buff *skb)
3680 {
3681         return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_INNOROUTES);
3682 }
3683
3684 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb)
3685 {
3686         skb->dev = skb_dst(skb)->dev;
3687         return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_OUTNOROUTES);
3688 }
3689
3690 /*
3691  *      Allocate a dst for local (unicast / anycast) address.
3692  */
3693
3694 struct fib6_info *addrconf_f6i_alloc(struct net *net,
3695                                      struct inet6_dev *idev,
3696                                      const struct in6_addr *addr,
3697                                      bool anycast, gfp_t gfp_flags)
3698 {
3699         u32 tb_id;
3700         struct net_device *dev = idev->dev;
3701         struct fib6_info *f6i;
3702
3703         f6i = fib6_info_alloc(gfp_flags);
3704         if (!f6i)
3705                 return ERR_PTR(-ENOMEM);
3706
3707         f6i->fib6_metrics = ip_fib_metrics_init(net, NULL, 0);
3708         f6i->dst_nocount = true;
3709         f6i->dst_host = true;
3710         f6i->fib6_protocol = RTPROT_KERNEL;
3711         f6i->fib6_flags = RTF_UP | RTF_NONEXTHOP;
3712         if (anycast) {
3713                 f6i->fib6_type = RTN_ANYCAST;
3714                 f6i->fib6_flags |= RTF_ANYCAST;
3715         } else {
3716                 f6i->fib6_type = RTN_LOCAL;
3717                 f6i->fib6_flags |= RTF_LOCAL;
3718         }
3719
3720         f6i->fib6_nh.nh_gw = *addr;
3721         dev_hold(dev);
3722         f6i->fib6_nh.nh_dev = dev;
3723         f6i->fib6_dst.addr = *addr;
3724         f6i->fib6_dst.plen = 128;
3725         tb_id = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL;
3726         f6i->fib6_table = fib6_get_table(net, tb_id);
3727
3728         return f6i;
3729 }
3730
3731 /* remove deleted ip from prefsrc entries */
3732 struct arg_dev_net_ip {
3733         struct net_device *dev;
3734         struct net *net;
3735         struct in6_addr *addr;
3736 };
3737
3738 static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg)
3739 {
3740         struct net_device *dev = ((struct arg_dev_net_ip *)arg)->dev;
3741         struct net *net = ((struct arg_dev_net_ip *)arg)->net;
3742         struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr;
3743
3744         if (((void *)rt->fib6_nh.nh_dev == dev || !dev) &&
3745             rt != net->ipv6.fib6_null_entry &&
3746             ipv6_addr_equal(addr, &rt->fib6_prefsrc.addr)) {
3747                 spin_lock_bh(&rt6_exception_lock);
3748                 /* remove prefsrc entry */
3749                 rt->fib6_prefsrc.plen = 0;
3750                 spin_unlock_bh(&rt6_exception_lock);
3751         }
3752         return 0;
3753 }
3754
3755 void rt6_remove_prefsrc(struct inet6_ifaddr *ifp)
3756 {
3757         struct net *net = dev_net(ifp->idev->dev);
3758         struct arg_dev_net_ip adni = {
3759                 .dev = ifp->idev->dev,
3760                 .net = net,
3761                 .addr = &ifp->addr,
3762         };
3763         fib6_clean_all(net, fib6_remove_prefsrc, &adni);
3764 }
3765
3766 #define RTF_RA_ROUTER           (RTF_ADDRCONF | RTF_DEFAULT | RTF_GATEWAY)
3767
3768 /* Remove routers and update dst entries when gateway turn into host. */
3769 static int fib6_clean_tohost(struct fib6_info *rt, void *arg)
3770 {
3771         struct in6_addr *gateway = (struct in6_addr *)arg;
3772
3773         if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) &&
3774             ipv6_addr_equal(gateway, &rt->fib6_nh.nh_gw)) {
3775                 return -1;
3776         }
3777
3778         /* Further clean up cached routes in exception table.
3779          * This is needed because cached route may have a different
3780          * gateway than its 'parent' in the case of an ip redirect.
3781          */
3782         rt6_exceptions_clean_tohost(rt, gateway);
3783
3784         return 0;
3785 }
3786
3787 void rt6_clean_tohost(struct net *net, struct in6_addr *gateway)
3788 {
3789         fib6_clean_all(net, fib6_clean_tohost, gateway);
3790 }
3791
3792 struct arg_netdev_event {
3793         const struct net_device *dev;
3794         union {
3795                 unsigned int nh_flags;
3796                 unsigned long event;
3797         };
3798 };
3799
3800 static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt)
3801 {
3802         struct fib6_info *iter;
3803         struct fib6_node *fn;
3804
3805         fn = rcu_dereference_protected(rt->fib6_node,
3806                         lockdep_is_held(&rt->fib6_table->tb6_lock));
3807         iter = rcu_dereference_protected(fn->leaf,
3808                         lockdep_is_held(&rt->fib6_table->tb6_lock));
3809         while (iter) {
3810                 if (iter->fib6_metric == rt->fib6_metric &&
3811                     rt6_qualify_for_ecmp(iter))
3812                         return iter;
3813                 iter = rcu_dereference_protected(iter->fib6_next,
3814                                 lockdep_is_held(&rt->fib6_table->tb6_lock));
3815         }
3816
3817         return NULL;
3818 }
3819
3820 static bool rt6_is_dead(const struct fib6_info *rt)
3821 {
3822         if (rt->fib6_nh.nh_flags & RTNH_F_DEAD ||
3823             (rt->fib6_nh.nh_flags & RTNH_F_LINKDOWN &&
3824              fib6_ignore_linkdown(rt)))
3825                 return true;
3826
3827         return false;
3828 }
3829
3830 static int rt6_multipath_total_weight(const struct fib6_info *rt)
3831 {
3832         struct fib6_info *iter;
3833         int total = 0;
3834
3835         if (!rt6_is_dead(rt))
3836                 total += rt->fib6_nh.nh_weight;
3837
3838         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
3839                 if (!rt6_is_dead(iter))
3840                         total += iter->fib6_nh.nh_weight;
3841         }
3842
3843         return total;
3844 }
3845
3846 static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total)
3847 {
3848         int upper_bound = -1;
3849
3850         if (!rt6_is_dead(rt)) {
3851                 *weight += rt->fib6_nh.nh_weight;
3852                 upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31,
3853                                                     total) - 1;
3854         }
3855         atomic_set(&rt->fib6_nh.nh_upper_bound, upper_bound);
3856 }
3857
3858 static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total)
3859 {
3860         struct fib6_info *iter;
3861         int weight = 0;
3862
3863         rt6_upper_bound_set(rt, &weight, total);
3864
3865         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
3866                 rt6_upper_bound_set(iter, &weight, total);
3867 }
3868
3869 void rt6_multipath_rebalance(struct fib6_info *rt)
3870 {
3871         struct fib6_info *first;
3872         int total;
3873
3874         /* In case the entire multipath route was marked for flushing,
3875          * then there is no need to rebalance upon the removal of every
3876          * sibling route.
3877          */
3878         if (!rt->fib6_nsiblings || rt->should_flush)
3879                 return;
3880
3881         /* During lookup routes are evaluated in order, so we need to
3882          * make sure upper bounds are assigned from the first sibling
3883          * onwards.
3884          */
3885         first = rt6_multipath_first_sibling(rt);
3886         if (WARN_ON_ONCE(!first))
3887                 return;
3888
3889         total = rt6_multipath_total_weight(first);
3890         rt6_multipath_upper_bound_set(first, total);
3891 }
3892
3893 static int fib6_ifup(struct fib6_info *rt, void *p_arg)
3894 {
3895         const struct arg_netdev_event *arg = p_arg;
3896         struct net *net = dev_net(arg->dev);
3897
3898         if (rt != net->ipv6.fib6_null_entry && rt->fib6_nh.nh_dev == arg->dev) {
3899                 rt->fib6_nh.nh_flags &= ~arg->nh_flags;
3900                 fib6_update_sernum_upto_root(net, rt);
3901                 rt6_multipath_rebalance(rt);
3902         }
3903
3904         return 0;
3905 }
3906
3907 void rt6_sync_up(struct net_device *dev, unsigned int nh_flags)
3908 {
3909         struct arg_netdev_event arg = {
3910                 .dev = dev,
3911                 {
3912                         .nh_flags = nh_flags,
3913                 },
3914         };
3915
3916         if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev))
3917                 arg.nh_flags |= RTNH_F_LINKDOWN;
3918
3919         fib6_clean_all(dev_net(dev), fib6_ifup, &arg);
3920 }
3921
3922 static bool rt6_multipath_uses_dev(const struct fib6_info *rt,
3923                                    const struct net_device *dev)
3924 {
3925         struct fib6_info *iter;
3926
3927         if (rt->fib6_nh.nh_dev == dev)
3928                 return true;
3929         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
3930                 if (iter->fib6_nh.nh_dev == dev)
3931                         return true;
3932
3933         return false;
3934 }
3935
3936 static void rt6_multipath_flush(struct fib6_info *rt)
3937 {
3938         struct fib6_info *iter;
3939
3940         rt->should_flush = 1;
3941         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
3942                 iter->should_flush = 1;
3943 }
3944
3945 static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt,
3946                                              const struct net_device *down_dev)
3947 {
3948         struct fib6_info *iter;
3949         unsigned int dead = 0;
3950
3951         if (rt->fib6_nh.nh_dev == down_dev ||
3952             rt->fib6_nh.nh_flags & RTNH_F_DEAD)
3953                 dead++;
3954         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
3955                 if (iter->fib6_nh.nh_dev == down_dev ||
3956                     iter->fib6_nh.nh_flags & RTNH_F_DEAD)
3957                         dead++;
3958
3959         return dead;
3960 }
3961
3962 static void rt6_multipath_nh_flags_set(struct fib6_info *rt,
3963                                        const struct net_device *dev,
3964                                        unsigned int nh_flags)
3965 {
3966         struct fib6_info *iter;
3967
3968         if (rt->fib6_nh.nh_dev == dev)
3969                 rt->fib6_nh.nh_flags |= nh_flags;
3970         list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
3971                 if (iter->fib6_nh.nh_dev == dev)
3972                         iter->fib6_nh.nh_flags |= nh_flags;
3973 }
3974
3975 /* called with write lock held for table with rt */
3976 static int fib6_ifdown(struct fib6_info *rt, void *p_arg)
3977 {
3978         const struct arg_netdev_event *arg = p_arg;
3979         const struct net_device *dev = arg->dev;
3980         struct net *net = dev_net(dev);
3981
3982         if (rt == net->ipv6.fib6_null_entry)
3983                 return 0;
3984
3985         switch (arg->event) {
3986         case NETDEV_UNREGISTER:
3987                 return rt->fib6_nh.nh_dev == dev ? -1 : 0;
3988         case NETDEV_DOWN:
3989                 if (rt->should_flush)
3990                         return -1;
3991                 if (!rt->fib6_nsiblings)
3992                         return rt->fib6_nh.nh_dev == dev ? -1 : 0;
3993                 if (rt6_multipath_uses_dev(rt, dev)) {
3994                         unsigned int count;
3995
3996                         count = rt6_multipath_dead_count(rt, dev);
3997                         if (rt->fib6_nsiblings + 1 == count) {
3998                                 rt6_multipath_flush(rt);
3999                                 return -1;
4000                         }
4001                         rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD |
4002                                                    RTNH_F_LINKDOWN);
4003                         fib6_update_sernum(net, rt);
4004                         rt6_multipath_rebalance(rt);
4005                 }
4006                 return -2;
4007         case NETDEV_CHANGE:
4008                 if (rt->fib6_nh.nh_dev != dev ||
4009                     rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
4010                         break;
4011                 rt->fib6_nh.nh_flags |= RTNH_F_LINKDOWN;
4012                 rt6_multipath_rebalance(rt);
4013                 break;
4014         }
4015
4016         return 0;
4017 }
4018
4019 void rt6_sync_down_dev(struct net_device *dev, unsigned long event)
4020 {
4021         struct arg_netdev_event arg = {
4022                 .dev = dev,
4023                 {
4024                         .event = event,
4025                 },
4026         };
4027
4028         fib6_clean_all(dev_net(dev), fib6_ifdown, &arg);
4029 }
4030
4031 void rt6_disable_ip(struct net_device *dev, unsigned long event)
4032 {
4033         rt6_sync_down_dev(dev, event);
4034         rt6_uncached_list_flush_dev(dev_net(dev), dev);
4035         neigh_ifdown(&nd_tbl, dev);
4036 }
4037
4038 struct rt6_mtu_change_arg {
4039         struct net_device *dev;
4040         unsigned int mtu;
4041 };
4042
4043 static int rt6_mtu_change_route(struct fib6_info *rt, void *p_arg)
4044 {
4045         struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg;
4046         struct inet6_dev *idev;
4047
4048         /* In IPv6 pmtu discovery is not optional,
4049            so that RTAX_MTU lock cannot disable it.
4050            We still use this lock to block changes
4051            caused by addrconf/ndisc.
4052         */
4053
4054         idev = __in6_dev_get(arg->dev);
4055         if (!idev)
4056                 return 0;
4057
4058         /* For administrative MTU increase, there is no way to discover
4059            IPv6 PMTU increase, so PMTU increase should be updated here.
4060            Since RFC 1981 doesn't include administrative MTU increase
4061            update PMTU increase is a MUST. (i.e. jumbo frame)
4062          */
4063         if (rt->fib6_nh.nh_dev == arg->dev &&
4064             !fib6_metric_locked(rt, RTAX_MTU)) {
4065                 u32 mtu = rt->fib6_pmtu;
4066
4067                 if (mtu >= arg->mtu ||
4068                     (mtu < arg->mtu && mtu == idev->cnf.mtu6))
4069                         fib6_metric_set(rt, RTAX_MTU, arg->mtu);
4070
4071                 spin_lock_bh(&rt6_exception_lock);
4072                 rt6_exceptions_update_pmtu(idev, rt, arg->mtu);
4073                 spin_unlock_bh(&rt6_exception_lock);
4074         }
4075         return 0;
4076 }
4077
4078 void rt6_mtu_change(struct net_device *dev, unsigned int mtu)
4079 {
4080         struct rt6_mtu_change_arg arg = {
4081                 .dev = dev,
4082                 .mtu = mtu,
4083         };
4084
4085         fib6_clean_all(dev_net(dev), rt6_mtu_change_route, &arg);
4086 }
4087
4088 static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = {
4089         [RTA_GATEWAY]           = { .len = sizeof(struct in6_addr) },
4090         [RTA_PREFSRC]           = { .len = sizeof(struct in6_addr) },
4091         [RTA_OIF]               = { .type = NLA_U32 },
4092         [RTA_IIF]               = { .type = NLA_U32 },
4093         [RTA_PRIORITY]          = { .type = NLA_U32 },
4094         [RTA_METRICS]           = { .type = NLA_NESTED },
4095         [RTA_MULTIPATH]         = { .len = sizeof(struct rtnexthop) },
4096         [RTA_PREF]              = { .type = NLA_U8 },
4097         [RTA_ENCAP_TYPE]        = { .type = NLA_U16 },
4098         [RTA_ENCAP]             = { .type = NLA_NESTED },
4099         [RTA_EXPIRES]           = { .type = NLA_U32 },
4100         [RTA_UID]               = { .type = NLA_U32 },
4101         [RTA_MARK]              = { .type = NLA_U32 },
4102         [RTA_TABLE]             = { .type = NLA_U32 },
4103         [RTA_IP_PROTO]          = { .type = NLA_U8 },
4104         [RTA_SPORT]             = { .type = NLA_U16 },
4105         [RTA_DPORT]             = { .type = NLA_U16 },
4106 };
4107
4108 static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh,
4109                               struct fib6_config *cfg,
4110                               struct netlink_ext_ack *extack)
4111 {
4112         struct rtmsg *rtm;
4113         struct nlattr *tb[RTA_MAX+1];
4114         unsigned int pref;
4115         int err;
4116
4117         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv6_policy,
4118                           NULL);
4119         if (err < 0)
4120                 goto errout;
4121
4122         err = -EINVAL;
4123         rtm = nlmsg_data(nlh);
4124
4125         *cfg = (struct fib6_config){
4126                 .fc_table = rtm->rtm_table,
4127                 .fc_dst_len = rtm->rtm_dst_len,
4128                 .fc_src_len = rtm->rtm_src_len,
4129                 .fc_flags = RTF_UP,
4130                 .fc_protocol = rtm->rtm_protocol,
4131                 .fc_type = rtm->rtm_type,
4132
4133                 .fc_nlinfo.portid = NETLINK_CB(skb).portid,
4134                 .fc_nlinfo.nlh = nlh,
4135                 .fc_nlinfo.nl_net = sock_net(skb->sk),
4136         };
4137
4138         if (rtm->rtm_type == RTN_UNREACHABLE ||
4139             rtm->rtm_type == RTN_BLACKHOLE ||
4140             rtm->rtm_type == RTN_PROHIBIT ||
4141             rtm->rtm_type == RTN_THROW)
4142                 cfg->fc_flags |= RTF_REJECT;
4143
4144         if (rtm->rtm_type == RTN_LOCAL)
4145                 cfg->fc_flags |= RTF_LOCAL;
4146
4147         if (rtm->rtm_flags & RTM_F_CLONED)
4148                 cfg->fc_flags |= RTF_CACHE;
4149
4150         cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK);
4151
4152         if (tb[RTA_GATEWAY]) {
4153                 cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]);
4154                 cfg->fc_flags |= RTF_GATEWAY;
4155         }
4156
4157         if (tb[RTA_DST]) {
4158                 int plen = (rtm->rtm_dst_len + 7) >> 3;
4159
4160                 if (nla_len(tb[RTA_DST]) < plen)
4161                         goto errout;
4162
4163                 nla_memcpy(&cfg->fc_dst, tb[RTA_DST], plen);
4164         }
4165
4166         if (tb[RTA_SRC]) {
4167                 int plen = (rtm->rtm_src_len + 7) >> 3;
4168
4169                 if (nla_len(tb[RTA_SRC]) < plen)
4170                         goto errout;
4171
4172                 nla_memcpy(&cfg->fc_src, tb[RTA_SRC], plen);
4173         }
4174
4175         if (tb[RTA_PREFSRC])
4176                 cfg->fc_prefsrc = nla_get_in6_addr(tb[RTA_PREFSRC]);
4177
4178         if (tb[RTA_OIF])
4179                 cfg->fc_ifindex = nla_get_u32(tb[RTA_OIF]);
4180
4181         if (tb[RTA_PRIORITY])
4182                 cfg->fc_metric = nla_get_u32(tb[RTA_PRIORITY]);
4183
4184         if (tb[RTA_METRICS]) {
4185                 cfg->fc_mx = nla_data(tb[RTA_METRICS]);
4186                 cfg->fc_mx_len = nla_len(tb[RTA_METRICS]);
4187         }
4188
4189         if (tb[RTA_TABLE])
4190                 cfg->fc_table = nla_get_u32(tb[RTA_TABLE]);
4191
4192         if (tb[RTA_MULTIPATH]) {
4193                 cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]);
4194                 cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]);
4195
4196                 err = lwtunnel_valid_encap_type_attr(cfg->fc_mp,
4197                                                      cfg->fc_mp_len, extack);
4198                 if (err < 0)
4199                         goto errout;
4200         }
4201
4202         if (tb[RTA_PREF]) {
4203                 pref = nla_get_u8(tb[RTA_PREF]);
4204                 if (pref != ICMPV6_ROUTER_PREF_LOW &&
4205                     pref != ICMPV6_ROUTER_PREF_HIGH)
4206                         pref = ICMPV6_ROUTER_PREF_MEDIUM;
4207                 cfg->fc_flags |= RTF_PREF(pref);
4208         }
4209
4210         if (tb[RTA_ENCAP])
4211                 cfg->fc_encap = tb[RTA_ENCAP];
4212
4213         if (tb[RTA_ENCAP_TYPE]) {
4214                 cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]);
4215
4216                 err = lwtunnel_valid_encap_type(cfg->fc_encap_type, extack);
4217                 if (err < 0)
4218                         goto errout;
4219         }
4220
4221         if (tb[RTA_EXPIRES]) {
4222                 unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ);
4223
4224                 if (addrconf_finite_timeout(timeout)) {
4225                         cfg->fc_expires = jiffies_to_clock_t(timeout * HZ);
4226                         cfg->fc_flags |= RTF_EXPIRES;
4227                 }
4228         }
4229
4230         err = 0;
4231 errout:
4232         return err;
4233 }
4234
4235 struct rt6_nh {
4236         struct fib6_info *fib6_info;
4237         struct fib6_config r_cfg;
4238         struct list_head next;
4239 };
4240
4241 static void ip6_print_replace_route_err(struct list_head *rt6_nh_list)
4242 {
4243         struct rt6_nh *nh;
4244
4245         list_for_each_entry(nh, rt6_nh_list, next) {
4246                 pr_warn("IPV6: multipath route replace failed (check consistency of installed routes): %pI6c nexthop %pI6c ifi %d\n",
4247                         &nh->r_cfg.fc_dst, &nh->r_cfg.fc_gateway,
4248                         nh->r_cfg.fc_ifindex);
4249         }
4250 }
4251
4252 static int ip6_route_info_append(struct net *net,
4253                                  struct list_head *rt6_nh_list,
4254                                  struct fib6_info *rt,
4255                                  struct fib6_config *r_cfg)
4256 {
4257         struct rt6_nh *nh;
4258         int err = -EEXIST;
4259
4260         list_for_each_entry(nh, rt6_nh_list, next) {
4261                 /* check if fib6_info already exists */
4262                 if (rt6_duplicate_nexthop(nh->fib6_info, rt))
4263                         return err;
4264         }
4265
4266         nh = kzalloc(sizeof(*nh), GFP_KERNEL);
4267         if (!nh)
4268                 return -ENOMEM;
4269         nh->fib6_info = rt;
4270         memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg));
4271         list_add_tail(&nh->next, rt6_nh_list);
4272
4273         return 0;
4274 }
4275
4276 static void ip6_route_mpath_notify(struct fib6_info *rt,
4277                                    struct fib6_info *rt_last,
4278                                    struct nl_info *info,
4279                                    __u16 nlflags)
4280 {
4281         /* if this is an APPEND route, then rt points to the first route
4282          * inserted and rt_last points to last route inserted. Userspace
4283          * wants a consistent dump of the route which starts at the first
4284          * nexthop. Since sibling routes are always added at the end of
4285          * the list, find the first sibling of the last route appended
4286          */
4287         if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) {
4288                 rt = list_first_entry(&rt_last->fib6_siblings,
4289                                       struct fib6_info,
4290                                       fib6_siblings);
4291         }
4292
4293         if (rt)
4294                 inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);
4295 }
4296
4297 static int ip6_route_multipath_add(struct fib6_config *cfg,
4298                                    struct netlink_ext_ack *extack)
4299 {
4300         struct fib6_info *rt_notif = NULL, *rt_last = NULL;
4301         struct nl_info *info = &cfg->fc_nlinfo;
4302         struct fib6_config r_cfg;
4303         struct rtnexthop *rtnh;
4304         struct fib6_info *rt;
4305         struct rt6_nh *err_nh;
4306         struct rt6_nh *nh, *nh_safe;
4307         __u16 nlflags;
4308         int remaining;
4309         int attrlen;
4310         int err = 1;
4311         int nhn = 0;
4312         int replace = (cfg->fc_nlinfo.nlh &&
4313                        (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE));
4314         LIST_HEAD(rt6_nh_list);
4315
4316         nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE;
4317         if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND)
4318                 nlflags |= NLM_F_APPEND;
4319
4320         remaining = cfg->fc_mp_len;
4321         rtnh = (struct rtnexthop *)cfg->fc_mp;
4322
4323         /* Parse a Multipath Entry and build a list (rt6_nh_list) of
4324          * fib6_info structs per nexthop
4325          */
4326         while (rtnh_ok(rtnh, remaining)) {
4327                 memcpy(&r_cfg, cfg, sizeof(*cfg));
4328                 if (rtnh->rtnh_ifindex)
4329                         r_cfg.fc_ifindex = rtnh->rtnh_ifindex;
4330
4331                 attrlen = rtnh_attrlen(rtnh);
4332                 if (attrlen > 0) {
4333                         struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
4334
4335                         nla = nla_find(attrs, attrlen, RTA_GATEWAY);
4336                         if (nla) {
4337                                 r_cfg.fc_gateway = nla_get_in6_addr(nla);
4338                                 r_cfg.fc_flags |= RTF_GATEWAY;
4339                         }
4340                         r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP);
4341                         nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE);
4342                         if (nla)
4343                                 r_cfg.fc_encap_type = nla_get_u16(nla);
4344                 }
4345
4346                 r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK);
4347                 rt = ip6_route_info_create(&r_cfg, GFP_KERNEL, extack);
4348                 if (IS_ERR(rt)) {
4349                         err = PTR_ERR(rt);
4350                         rt = NULL;
4351                         goto cleanup;
4352                 }
4353                 if (!rt6_qualify_for_ecmp(rt)) {
4354                         err = -EINVAL;
4355                         NL_SET_ERR_MSG(extack,
4356                                        "Device only routes can not be added for IPv6 using the multipath API.");
4357                         fib6_info_release(rt);
4358                         goto cleanup;
4359                 }
4360
4361                 rt->fib6_nh.nh_weight = rtnh->rtnh_hops + 1;
4362
4363                 err = ip6_route_info_append(info->nl_net, &rt6_nh_list,
4364                                             rt, &r_cfg);
4365                 if (err) {
4366                         fib6_info_release(rt);
4367                         goto cleanup;
4368                 }
4369
4370                 rtnh = rtnh_next(rtnh, &remaining);
4371         }
4372
4373         /* for add and replace send one notification with all nexthops.
4374          * Skip the notification in fib6_add_rt2node and send one with
4375          * the full route when done
4376          */
4377         info->skip_notify = 1;
4378
4379         err_nh = NULL;
4380         list_for_each_entry(nh, &rt6_nh_list, next) {
4381                 err = __ip6_ins_rt(nh->fib6_info, info, extack);
4382                 fib6_info_release(nh->fib6_info);
4383
4384                 if (!err) {
4385                         /* save reference to last route successfully inserted */
4386                         rt_last = nh->fib6_info;
4387
4388                         /* save reference to first route for notification */
4389                         if (!rt_notif)
4390                                 rt_notif = nh->fib6_info;
4391                 }
4392
4393                 /* nh->fib6_info is used or freed at this point, reset to NULL*/
4394                 nh->fib6_info = NULL;
4395                 if (err) {
4396                         if (replace && nhn)
4397                                 ip6_print_replace_route_err(&rt6_nh_list);
4398                         err_nh = nh;
4399                         goto add_errout;
4400                 }
4401
4402                 /* Because each route is added like a single route we remove
4403                  * these flags after the first nexthop: if there is a collision,
4404                  * we have already failed to add the first nexthop:
4405                  * fib6_add_rt2node() has rejected it; when replacing, old
4406                  * nexthops have been replaced by first new, the rest should
4407                  * be added to it.
4408                  */
4409                 cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL |
4410                                                      NLM_F_REPLACE);
4411                 nhn++;
4412         }
4413
4414         /* success ... tell user about new route */
4415         ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags);
4416         goto cleanup;
4417
4418 add_errout:
4419         /* send notification for routes that were added so that
4420          * the delete notifications sent by ip6_route_del are
4421          * coherent
4422          */
4423         if (rt_notif)
4424                 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags);
4425
4426         /* Delete routes that were already added */
4427         list_for_each_entry(nh, &rt6_nh_list, next) {
4428                 if (err_nh == nh)
4429                         break;
4430                 ip6_route_del(&nh->r_cfg, extack);
4431         }
4432
4433 cleanup:
4434         list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) {
4435                 if (nh->fib6_info)
4436                         fib6_info_release(nh->fib6_info);
4437                 list_del(&nh->next);
4438                 kfree(nh);
4439         }
4440
4441         return err;
4442 }
4443
4444 static int ip6_route_multipath_del(struct fib6_config *cfg,
4445                                    struct netlink_ext_ack *extack)
4446 {
4447         struct fib6_config r_cfg;
4448         struct rtnexthop *rtnh;
4449         int remaining;
4450         int attrlen;
4451         int err = 1, last_err = 0;
4452
4453         remaining = cfg->fc_mp_len;
4454         rtnh = (struct rtnexthop *)cfg->fc_mp;
4455
4456         /* Parse a Multipath Entry */
4457         while (rtnh_ok(rtnh, remaining)) {
4458                 memcpy(&r_cfg, cfg, sizeof(*cfg));
4459                 if (rtnh->rtnh_ifindex)
4460                         r_cfg.fc_ifindex = rtnh->rtnh_ifindex;
4461
4462                 attrlen = rtnh_attrlen(rtnh);
4463                 if (attrlen > 0) {
4464                         struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
4465
4466                         nla = nla_find(attrs, attrlen, RTA_GATEWAY);
4467                         if (nla) {
4468                                 nla_memcpy(&r_cfg.fc_gateway, nla, 16);
4469                                 r_cfg.fc_flags |= RTF_GATEWAY;
4470                         }
4471                 }
4472                 err = ip6_route_del(&r_cfg, extack);
4473                 if (err)
4474                         last_err = err;
4475
4476                 rtnh = rtnh_next(rtnh, &remaining);
4477         }
4478
4479         return last_err;
4480 }
4481
4482 static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
4483                               struct netlink_ext_ack *extack)
4484 {
4485         struct fib6_config cfg;
4486         int err;
4487
4488         err = rtm_to_fib6_config(skb, nlh, &cfg, extack);
4489         if (err < 0)
4490                 return err;
4491
4492         if (cfg.fc_mp)
4493                 return ip6_route_multipath_del(&cfg, extack);
4494         else {
4495                 cfg.fc_delete_all_nh = 1;
4496                 return ip6_route_del(&cfg, extack);
4497         }
4498 }
4499
4500 static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
4501                               struct netlink_ext_ack *extack)
4502 {
4503         struct fib6_config cfg;
4504         int err;
4505
4506         err = rtm_to_fib6_config(skb, nlh, &cfg, extack);
4507         if (err < 0)
4508                 return err;
4509
4510         if (cfg.fc_mp)
4511                 return ip6_route_multipath_add(&cfg, extack);
4512         else
4513                 return ip6_route_add(&cfg, GFP_KERNEL, extack);
4514 }
4515
4516 static size_t rt6_nlmsg_size(struct fib6_info *rt)
4517 {
4518         int nexthop_len = 0;
4519
4520         if (rt->fib6_nsiblings) {
4521                 nexthop_len = nla_total_size(0)  /* RTA_MULTIPATH */
4522                             + NLA_ALIGN(sizeof(struct rtnexthop))
4523                             + nla_total_size(16) /* RTA_GATEWAY */
4524                             + lwtunnel_get_encap_size(rt->fib6_nh.nh_lwtstate);
4525
4526                 nexthop_len *= rt->fib6_nsiblings;
4527         }
4528
4529         return NLMSG_ALIGN(sizeof(struct rtmsg))
4530                + nla_total_size(16) /* RTA_SRC */
4531                + nla_total_size(16) /* RTA_DST */
4532                + nla_total_size(16) /* RTA_GATEWAY */
4533                + nla_total_size(16) /* RTA_PREFSRC */
4534                + nla_total_size(4) /* RTA_TABLE */
4535                + nla_total_size(4) /* RTA_IIF */
4536                + nla_total_size(4) /* RTA_OIF */
4537                + nla_total_size(4) /* RTA_PRIORITY */
4538                + RTAX_MAX * nla_total_size(4) /* RTA_METRICS */
4539                + nla_total_size(sizeof(struct rta_cacheinfo))
4540                + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */
4541                + nla_total_size(1) /* RTA_PREF */
4542                + lwtunnel_get_encap_size(rt->fib6_nh.nh_lwtstate)
4543                + nexthop_len;
4544 }
4545
4546 static int rt6_nexthop_info(struct sk_buff *skb, struct fib6_info *rt,
4547                             unsigned int *flags, bool skip_oif)
4548 {
4549         if (rt->fib6_nh.nh_flags & RTNH_F_DEAD)
4550                 *flags |= RTNH_F_DEAD;
4551
4552         if (rt->fib6_nh.nh_flags & RTNH_F_LINKDOWN) {
4553                 *flags |= RTNH_F_LINKDOWN;
4554
4555                 rcu_read_lock();
4556                 if (fib6_ignore_linkdown(rt))
4557                         *flags |= RTNH_F_DEAD;
4558                 rcu_read_unlock();
4559         }
4560
4561         if (rt->fib6_flags & RTF_GATEWAY) {
4562                 if (nla_put_in6_addr(skb, RTA_GATEWAY, &rt->fib6_nh.nh_gw) < 0)
4563                         goto nla_put_failure;
4564         }
4565
4566         *flags |= (rt->fib6_nh.nh_flags & RTNH_F_ONLINK);
4567         if (rt->fib6_nh.nh_flags & RTNH_F_OFFLOAD)
4568                 *flags |= RTNH_F_OFFLOAD;
4569
4570         /* not needed for multipath encoding b/c it has a rtnexthop struct */
4571         if (!skip_oif && rt->fib6_nh.nh_dev &&
4572             nla_put_u32(skb, RTA_OIF, rt->fib6_nh.nh_dev->ifindex))
4573                 goto nla_put_failure;
4574
4575         if (rt->fib6_nh.nh_lwtstate &&
4576             lwtunnel_fill_encap(skb, rt->fib6_nh.nh_lwtstate) < 0)
4577                 goto nla_put_failure;
4578
4579         return 0;
4580
4581 nla_put_failure:
4582         return -EMSGSIZE;
4583 }
4584
4585 /* add multipath next hop */
4586 static int rt6_add_nexthop(struct sk_buff *skb, struct fib6_info *rt)
4587 {
4588         const struct net_device *dev = rt->fib6_nh.nh_dev;
4589         struct rtnexthop *rtnh;
4590         unsigned int flags = 0;
4591
4592         rtnh = nla_reserve_nohdr(skb, sizeof(*rtnh));
4593         if (!rtnh)
4594                 goto nla_put_failure;
4595
4596         rtnh->rtnh_hops = rt->fib6_nh.nh_weight - 1;
4597         rtnh->rtnh_ifindex = dev ? dev->ifindex : 0;
4598
4599         if (rt6_nexthop_info(skb, rt, &flags, true) < 0)
4600                 goto nla_put_failure;
4601
4602         rtnh->rtnh_flags = flags;
4603
4604         /* length of rtnetlink header + attributes */
4605         rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *)rtnh;
4606
4607         return 0;
4608
4609 nla_put_failure:
4610         return -EMSGSIZE;
4611 }
4612
4613 static int rt6_fill_node(struct net *net, struct sk_buff *skb,
4614                          struct fib6_info *rt, struct dst_entry *dst,
4615                          struct in6_addr *dest, struct in6_addr *src,
4616                          int iif, int type, u32 portid, u32 seq,
4617                          unsigned int flags)
4618 {
4619         struct rt6_info *rt6 = (struct rt6_info *)dst;
4620         struct rt6key *rt6_dst, *rt6_src;
4621         u32 *pmetrics, table, rt6_flags;
4622         struct nlmsghdr *nlh;
4623         struct rtmsg *rtm;
4624         long expires = 0;
4625
4626         nlh = nlmsg_put(skb, portid, seq, type, sizeof(*rtm), flags);
4627         if (!nlh)
4628                 return -EMSGSIZE;
4629
4630         if (rt6) {
4631                 rt6_dst = &rt6->rt6i_dst;
4632                 rt6_src = &rt6->rt6i_src;
4633                 rt6_flags = rt6->rt6i_flags;
4634         } else {
4635                 rt6_dst = &rt->fib6_dst;
4636                 rt6_src = &rt->fib6_src;
4637                 rt6_flags = rt->fib6_flags;
4638         }
4639
4640         rtm = nlmsg_data(nlh);
4641         rtm->rtm_family = AF_INET6;
4642         rtm->rtm_dst_len = rt6_dst->plen;
4643         rtm->rtm_src_len = rt6_src->plen;
4644         rtm->rtm_tos = 0;
4645         if (rt->fib6_table)
4646                 table = rt->fib6_table->tb6_id;
4647         else
4648                 table = RT6_TABLE_UNSPEC;
4649         rtm->rtm_table = table;
4650         if (nla_put_u32(skb, RTA_TABLE, table))
4651                 goto nla_put_failure;
4652
4653         rtm->rtm_type = rt->fib6_type;
4654         rtm->rtm_flags = 0;
4655         rtm->rtm_scope = RT_SCOPE_UNIVERSE;
4656         rtm->rtm_protocol = rt->fib6_protocol;
4657
4658         if (rt6_flags & RTF_CACHE)
4659                 rtm->rtm_flags |= RTM_F_CLONED;
4660
4661         if (dest) {
4662                 if (nla_put_in6_addr(skb, RTA_DST, dest))
4663                         goto nla_put_failure;
4664                 rtm->rtm_dst_len = 128;
4665         } else if (rtm->rtm_dst_len)
4666                 if (nla_put_in6_addr(skb, RTA_DST, &rt6_dst->addr))
4667                         goto nla_put_failure;
4668 #ifdef CONFIG_IPV6_SUBTREES
4669         if (src) {
4670                 if (nla_put_in6_addr(skb, RTA_SRC, src))
4671                         goto nla_put_failure;
4672                 rtm->rtm_src_len = 128;
4673         } else if (rtm->rtm_src_len &&
4674                    nla_put_in6_addr(skb, RTA_SRC, &rt6_src->addr))
4675                 goto nla_put_failure;
4676 #endif
4677         if (iif) {
4678 #ifdef CONFIG_IPV6_MROUTE
4679                 if (ipv6_addr_is_multicast(&rt6_dst->addr)) {
4680                         int err = ip6mr_get_route(net, skb, rtm, portid);
4681
4682                         if (err == 0)
4683                                 return 0;
4684                         if (err < 0)
4685                                 goto nla_put_failure;
4686                 } else
4687 #endif
4688                         if (nla_put_u32(skb, RTA_IIF, iif))
4689                                 goto nla_put_failure;
4690         } else if (dest) {
4691                 struct in6_addr saddr_buf;
4692                 if (ip6_route_get_saddr(net, rt, dest, 0, &saddr_buf) == 0 &&
4693                     nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf))
4694                         goto nla_put_failure;
4695         }
4696
4697         if (rt->fib6_prefsrc.plen) {
4698                 struct in6_addr saddr_buf;
4699                 saddr_buf = rt->fib6_prefsrc.addr;
4700                 if (nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf))
4701                         goto nla_put_failure;
4702         }
4703
4704         pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics;
4705         if (rtnetlink_put_metrics(skb, pmetrics) < 0)
4706                 goto nla_put_failure;
4707
4708         if (nla_put_u32(skb, RTA_PRIORITY, rt->fib6_metric))
4709                 goto nla_put_failure;
4710
4711         /* For multipath routes, walk the siblings list and add
4712          * each as a nexthop within RTA_MULTIPATH.
4713          */
4714         if (rt6) {
4715                 if (rt6_flags & RTF_GATEWAY &&
4716                     nla_put_in6_addr(skb, RTA_GATEWAY, &rt6->rt6i_gateway))
4717                         goto nla_put_failure;
4718
4719                 if (dst->dev && nla_put_u32(skb, RTA_OIF, dst->dev->ifindex))
4720                         goto nla_put_failure;
4721         } else if (rt->fib6_nsiblings) {
4722                 struct fib6_info *sibling, *next_sibling;
4723                 struct nlattr *mp;
4724
4725                 mp = nla_nest_start(skb, RTA_MULTIPATH);
4726                 if (!mp)
4727                         goto nla_put_failure;
4728
4729                 if (rt6_add_nexthop(skb, rt) < 0)
4730                         goto nla_put_failure;
4731
4732                 list_for_each_entry_safe(sibling, next_sibling,
4733                                          &rt->fib6_siblings, fib6_siblings) {
4734                         if (rt6_add_nexthop(skb, sibling) < 0)
4735                                 goto nla_put_failure;
4736                 }
4737
4738                 nla_nest_end(skb, mp);
4739         } else {
4740                 if (rt6_nexthop_info(skb, rt, &rtm->rtm_flags, false) < 0)
4741                         goto nla_put_failure;
4742         }
4743
4744         if (rt6_flags & RTF_EXPIRES) {
4745                 expires = dst ? dst->expires : rt->expires;
4746                 expires -= jiffies;
4747         }
4748
4749         if (rtnl_put_cacheinfo(skb, dst, 0, expires, dst ? dst->error : 0) < 0)
4750                 goto nla_put_failure;
4751
4752         if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt6_flags)))
4753                 goto nla_put_failure;
4754
4755
4756         nlmsg_end(skb, nlh);
4757         return 0;
4758
4759 nla_put_failure:
4760         nlmsg_cancel(skb, nlh);
4761         return -EMSGSIZE;
4762 }
4763
4764 int rt6_dump_route(struct fib6_info *rt, void *p_arg)
4765 {
4766         struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg;
4767         struct net *net = arg->net;
4768
4769         if (rt == net->ipv6.fib6_null_entry)
4770                 return 0;
4771
4772         if (nlmsg_len(arg->cb->nlh) >= sizeof(struct rtmsg)) {
4773                 struct rtmsg *rtm = nlmsg_data(arg->cb->nlh);
4774
4775                 /* user wants prefix routes only */
4776                 if (rtm->rtm_flags & RTM_F_PREFIX &&
4777                     !(rt->fib6_flags & RTF_PREFIX_RT)) {
4778                         /* success since this is not a prefix route */
4779                         return 1;
4780                 }
4781         }
4782
4783         return rt6_fill_node(net, arg->skb, rt, NULL, NULL, NULL, 0,
4784                              RTM_NEWROUTE, NETLINK_CB(arg->cb->skb).portid,
4785                              arg->cb->nlh->nlmsg_seq, NLM_F_MULTI);
4786 }
4787
4788 static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
4789                               struct netlink_ext_ack *extack)
4790 {
4791         struct net *net = sock_net(in_skb->sk);
4792         struct nlattr *tb[RTA_MAX+1];
4793         int err, iif = 0, oif = 0;
4794         struct fib6_info *from;
4795         struct dst_entry *dst;
4796         struct rt6_info *rt;
4797         struct sk_buff *skb;
4798         struct rtmsg *rtm;
4799         struct flowi6 fl6 = {};
4800         bool fibmatch;
4801
4802         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv6_policy,
4803                           extack);
4804         if (err < 0)
4805                 goto errout;
4806
4807         err = -EINVAL;
4808         rtm = nlmsg_data(nlh);
4809         fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, 0);
4810         fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH);
4811
4812         if (tb[RTA_SRC]) {
4813                 if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr))
4814                         goto errout;
4815
4816                 fl6.saddr = *(struct in6_addr *)nla_data(tb[RTA_SRC]);
4817         }
4818
4819         if (tb[RTA_DST]) {
4820                 if (nla_len(tb[RTA_DST]) < sizeof(struct in6_addr))
4821                         goto errout;
4822
4823                 fl6.daddr = *(struct in6_addr *)nla_data(tb[RTA_DST]);
4824         }
4825
4826         if (tb[RTA_IIF])
4827                 iif = nla_get_u32(tb[RTA_IIF]);
4828
4829         if (tb[RTA_OIF])
4830                 oif = nla_get_u32(tb[RTA_OIF]);
4831
4832         if (tb[RTA_MARK])
4833                 fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]);
4834
4835         if (tb[RTA_UID])
4836                 fl6.flowi6_uid = make_kuid(current_user_ns(),
4837                                            nla_get_u32(tb[RTA_UID]));
4838         else
4839                 fl6.flowi6_uid = iif ? INVALID_UID : current_uid();
4840
4841         if (tb[RTA_SPORT])
4842                 fl6.fl6_sport = nla_get_be16(tb[RTA_SPORT]);
4843
4844         if (tb[RTA_DPORT])
4845                 fl6.fl6_dport = nla_get_be16(tb[RTA_DPORT]);
4846
4847         if (tb[RTA_IP_PROTO]) {
4848                 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
4849                                                   &fl6.flowi6_proto, extack);
4850                 if (err)
4851                         goto errout;
4852         }
4853
4854         if (iif) {
4855                 struct net_device *dev;
4856                 int flags = 0;
4857
4858                 rcu_read_lock();
4859
4860                 dev = dev_get_by_index_rcu(net, iif);
4861                 if (!dev) {
4862                         rcu_read_unlock();
4863                         err = -ENODEV;
4864                         goto errout;
4865                 }
4866
4867                 fl6.flowi6_iif = iif;
4868
4869                 if (!ipv6_addr_any(&fl6.saddr))
4870                         flags |= RT6_LOOKUP_F_HAS_SADDR;
4871
4872                 dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags);
4873
4874                 rcu_read_unlock();
4875         } else {
4876                 fl6.flowi6_oif = oif;
4877
4878                 dst = ip6_route_output(net, NULL, &fl6);
4879         }
4880
4881
4882         rt = container_of(dst, struct rt6_info, dst);
4883         if (rt->dst.error) {
4884                 err = rt->dst.error;
4885                 ip6_rt_put(rt);
4886                 goto errout;
4887         }
4888
4889         if (rt == net->ipv6.ip6_null_entry) {
4890                 err = rt->dst.error;
4891                 ip6_rt_put(rt);
4892                 goto errout;
4893         }
4894
4895         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
4896         if (!skb) {
4897                 ip6_rt_put(rt);
4898                 err = -ENOBUFS;
4899                 goto errout;
4900         }
4901
4902         skb_dst_set(skb, &rt->dst);
4903
4904         rcu_read_lock();
4905         from = rcu_dereference(rt->from);
4906
4907         if (fibmatch)
4908                 err = rt6_fill_node(net, skb, from, NULL, NULL, NULL, iif,
4909                                     RTM_NEWROUTE, NETLINK_CB(in_skb).portid,
4910                                     nlh->nlmsg_seq, 0);
4911         else
4912                 err = rt6_fill_node(net, skb, from, dst, &fl6.daddr,
4913                                     &fl6.saddr, iif, RTM_NEWROUTE,
4914                                     NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
4915                                     0);
4916         rcu_read_unlock();
4917
4918         if (err < 0) {
4919                 kfree_skb(skb);
4920                 goto errout;
4921         }
4922
4923         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
4924 errout:
4925         return err;
4926 }
4927
4928 void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info,
4929                      unsigned int nlm_flags)
4930 {
4931         struct sk_buff *skb;
4932         struct net *net = info->nl_net;
4933         u32 seq;
4934         int err;
4935
4936         err = -ENOBUFS;
4937         seq = info->nlh ? info->nlh->nlmsg_seq : 0;
4938
4939         skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
4940         if (!skb)
4941                 goto errout;
4942
4943         err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0,
4944                             event, info->portid, seq, nlm_flags);
4945         if (err < 0) {
4946                 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
4947                 WARN_ON(err == -EMSGSIZE);
4948                 kfree_skb(skb);
4949                 goto errout;
4950         }
4951         rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
4952                     info->nlh, gfp_any());
4953         return;
4954 errout:
4955         if (err < 0)
4956                 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
4957 }
4958
4959 static int ip6_route_dev_notify(struct notifier_block *this,
4960                                 unsigned long event, void *ptr)
4961 {
4962         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
4963         struct net *net = dev_net(dev);
4964
4965         if (!(dev->flags & IFF_LOOPBACK))
4966                 return NOTIFY_OK;
4967
4968         if (event == NETDEV_REGISTER) {
4969                 net->ipv6.fib6_null_entry->fib6_nh.nh_dev = dev;
4970                 net->ipv6.ip6_null_entry->dst.dev = dev;
4971                 net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev);
4972 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
4973                 net->ipv6.ip6_prohibit_entry->dst.dev = dev;
4974                 net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev);
4975                 net->ipv6.ip6_blk_hole_entry->dst.dev = dev;
4976                 net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev);
4977 #endif
4978          } else if (event == NETDEV_UNREGISTER &&
4979                     dev->reg_state != NETREG_UNREGISTERED) {
4980                 /* NETDEV_UNREGISTER could be fired for multiple times by
4981                  * netdev_wait_allrefs(). Make sure we only call this once.
4982                  */
4983                 in6_dev_put_clear(&net->ipv6.ip6_null_entry->rt6i_idev);
4984 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
4985                 in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev);
4986                 in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev);
4987 #endif
4988         }
4989
4990         return NOTIFY_OK;
4991 }
4992
4993 /*
4994  *      /proc
4995  */
4996
4997 #ifdef CONFIG_PROC_FS
4998 static int rt6_stats_seq_show(struct seq_file *seq, void *v)
4999 {
5000         struct net *net = (struct net *)seq->private;
5001         seq_printf(seq, "%04x %04x %04x %04x %04x %04x %04x\n",
5002                    net->ipv6.rt6_stats->fib_nodes,
5003                    net->ipv6.rt6_stats->fib_route_nodes,
5004                    atomic_read(&net->ipv6.rt6_stats->fib_rt_alloc),
5005                    net->ipv6.rt6_stats->fib_rt_entries,
5006                    net->ipv6.rt6_stats->fib_rt_cache,
5007                    dst_entries_get_slow(&net->ipv6.ip6_dst_ops),
5008                    net->ipv6.rt6_stats->fib_discarded_routes);
5009
5010         return 0;
5011 }
5012 #endif  /* CONFIG_PROC_FS */
5013
5014 #ifdef CONFIG_SYSCTL
5015
5016 static
5017 int ipv6_sysctl_rtcache_flush(struct ctl_table *ctl, int write,
5018                               void __user *buffer, size_t *lenp, loff_t *ppos)
5019 {
5020         struct net *net;
5021         int delay;
5022         if (!write)
5023                 return -EINVAL;
5024
5025         net = (struct net *)ctl->extra1;
5026         delay = net->ipv6.sysctl.flush_delay;
5027         proc_dointvec(ctl, write, buffer, lenp, ppos);
5028         fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0);
5029         return 0;
5030 }
5031
5032 struct ctl_table ipv6_route_table_template[] = {
5033         {
5034                 .procname       =       "flush",
5035                 .data           =       &init_net.ipv6.sysctl.flush_delay,
5036                 .maxlen         =       sizeof(int),
5037                 .mode           =       0200,
5038                 .proc_handler   =       ipv6_sysctl_rtcache_flush
5039         },
5040         {
5041                 .procname       =       "gc_thresh",
5042                 .data           =       &ip6_dst_ops_template.gc_thresh,
5043                 .maxlen         =       sizeof(int),
5044                 .mode           =       0644,
5045                 .proc_handler   =       proc_dointvec,
5046         },
5047         {
5048                 .procname       =       "max_size",
5049                 .data           =       &init_net.ipv6.sysctl.ip6_rt_max_size,
5050                 .maxlen         =       sizeof(int),
5051                 .mode           =       0644,
5052                 .proc_handler   =       proc_dointvec,
5053         },
5054         {
5055                 .procname       =       "gc_min_interval",
5056                 .data           =       &init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
5057                 .maxlen         =       sizeof(int),
5058                 .mode           =       0644,
5059                 .proc_handler   =       proc_dointvec_jiffies,
5060         },
5061         {
5062                 .procname       =       "gc_timeout",
5063                 .data           =       &init_net.ipv6.sysctl.ip6_rt_gc_timeout,
5064                 .maxlen         =       sizeof(int),
5065                 .mode           =       0644,
5066                 .proc_handler   =       proc_dointvec_jiffies,
5067         },
5068         {
5069                 .procname       =       "gc_interval",
5070                 .data           =       &init_net.ipv6.sysctl.ip6_rt_gc_interval,
5071                 .maxlen         =       sizeof(int),
5072                 .mode           =       0644,
5073                 .proc_handler   =       proc_dointvec_jiffies,
5074         },
5075         {
5076                 .procname       =       "gc_elasticity",
5077                 .data           =       &init_net.ipv6.sysctl.ip6_rt_gc_elasticity,
5078                 .maxlen         =       sizeof(int),
5079                 .mode           =       0644,
5080                 .proc_handler   =       proc_dointvec,
5081         },
5082         {
5083                 .procname       =       "mtu_expires",
5084                 .data           =       &init_net.ipv6.sysctl.ip6_rt_mtu_expires,
5085                 .maxlen         =       sizeof(int),
5086                 .mode           =       0644,
5087                 .proc_handler   =       proc_dointvec_jiffies,
5088         },
5089         {
5090                 .procname       =       "min_adv_mss",
5091                 .data           =       &init_net.ipv6.sysctl.ip6_rt_min_advmss,
5092                 .maxlen         =       sizeof(int),
5093                 .mode           =       0644,
5094                 .proc_handler   =       proc_dointvec,
5095         },
5096         {
5097                 .procname       =       "gc_min_interval_ms",
5098                 .data           =       &init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
5099                 .maxlen         =       sizeof(int),
5100                 .mode           =       0644,
5101                 .proc_handler   =       proc_dointvec_ms_jiffies,
5102         },
5103         { }
5104 };
5105
5106 struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net)
5107 {
5108         struct ctl_table *table;
5109
5110         table = kmemdup(ipv6_route_table_template,
5111                         sizeof(ipv6_route_table_template),
5112                         GFP_KERNEL);
5113
5114         if (table) {
5115                 table[0].data = &net->ipv6.sysctl.flush_delay;
5116                 table[0].extra1 = net;
5117                 table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh;
5118                 table[2].data = &net->ipv6.sysctl.ip6_rt_max_size;
5119                 table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
5120                 table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout;
5121                 table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval;
5122                 table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity;
5123                 table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires;
5124                 table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss;
5125                 table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
5126
5127                 /* Don't export sysctls to unprivileged users */
5128                 if (net->user_ns != &init_user_ns)
5129                         table[0].procname = NULL;
5130         }
5131
5132         return table;
5133 }
5134 #endif
5135
5136 static int __net_init ip6_route_net_init(struct net *net)
5137 {
5138         int ret = -ENOMEM;
5139
5140         memcpy(&net->ipv6.ip6_dst_ops, &ip6_dst_ops_template,
5141                sizeof(net->ipv6.ip6_dst_ops));
5142
5143         if (dst_entries_init(&net->ipv6.ip6_dst_ops) < 0)
5144                 goto out_ip6_dst_ops;
5145
5146         net->ipv6.fib6_null_entry = kmemdup(&fib6_null_entry_template,
5147                                             sizeof(*net->ipv6.fib6_null_entry),
5148                                             GFP_KERNEL);
5149         if (!net->ipv6.fib6_null_entry)
5150                 goto out_ip6_dst_entries;
5151
5152         net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template,
5153                                            sizeof(*net->ipv6.ip6_null_entry),
5154                                            GFP_KERNEL);
5155         if (!net->ipv6.ip6_null_entry)
5156                 goto out_fib6_null_entry;
5157         net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops;
5158         dst_init_metrics(&net->ipv6.ip6_null_entry->dst,
5159                          ip6_template_metrics, true);
5160
5161 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
5162         net->ipv6.fib6_has_custom_rules = false;
5163         net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template,
5164                                                sizeof(*net->ipv6.ip6_prohibit_entry),
5165                                                GFP_KERNEL);
5166         if (!net->ipv6.ip6_prohibit_entry)
5167                 goto out_ip6_null_entry;
5168         net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops;
5169         dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst,
5170                          ip6_template_metrics, true);
5171
5172         net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template,
5173                                                sizeof(*net->ipv6.ip6_blk_hole_entry),
5174                                                GFP_KERNEL);
5175         if (!net->ipv6.ip6_blk_hole_entry)
5176                 goto out_ip6_prohibit_entry;
5177         net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops;
5178         dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst,
5179                          ip6_template_metrics, true);
5180 #endif
5181
5182         net->ipv6.sysctl.flush_delay = 0;
5183         net->ipv6.sysctl.ip6_rt_max_size = 4096;
5184         net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
5185         net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
5186         net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
5187         net->ipv6.sysctl.ip6_rt_gc_elasticity = 9;
5188         net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ;
5189         net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40;
5190
5191         net->ipv6.ip6_rt_gc_expire = 30*HZ;
5192
5193         ret = 0;
5194 out:
5195         return ret;
5196
5197 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
5198 out_ip6_prohibit_entry:
5199         kfree(net->ipv6.ip6_prohibit_entry);
5200 out_ip6_null_entry:
5201         kfree(net->ipv6.ip6_null_entry);
5202 #endif
5203 out_fib6_null_entry:
5204         kfree(net->ipv6.fib6_null_entry);
5205 out_ip6_dst_entries:
5206         dst_entries_destroy(&net->ipv6.ip6_dst_ops);
5207 out_ip6_dst_ops:
5208         goto out;
5209 }
5210
5211 static void __net_exit ip6_route_net_exit(struct net *net)
5212 {
5213         kfree(net->ipv6.fib6_null_entry);
5214         kfree(net->ipv6.ip6_null_entry);
5215 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
5216         kfree(net->ipv6.ip6_prohibit_entry);
5217         kfree(net->ipv6.ip6_blk_hole_entry);
5218 #endif
5219         dst_entries_destroy(&net->ipv6.ip6_dst_ops);
5220 }
5221
5222 static int __net_init ip6_route_net_init_late(struct net *net)
5223 {
5224 #ifdef CONFIG_PROC_FS
5225         proc_create_net("ipv6_route", 0, net->proc_net, &ipv6_route_seq_ops,
5226                         sizeof(struct ipv6_route_iter));
5227         proc_create_net_single("rt6_stats", 0444, net->proc_net,
5228                         rt6_stats_seq_show, NULL);
5229 #endif
5230         return 0;
5231 }
5232
5233 static void __net_exit ip6_route_net_exit_late(struct net *net)
5234 {
5235 #ifdef CONFIG_PROC_FS
5236         remove_proc_entry("ipv6_route", net->proc_net);
5237         remove_proc_entry("rt6_stats", net->proc_net);
5238 #endif
5239 }
5240
5241 static struct pernet_operations ip6_route_net_ops = {
5242         .init = ip6_route_net_init,
5243         .exit = ip6_route_net_exit,
5244 };
5245
5246 static int __net_init ipv6_inetpeer_init(struct net *net)
5247 {
5248         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
5249
5250         if (!bp)
5251                 return -ENOMEM;
5252         inet_peer_base_init(bp);
5253         net->ipv6.peers = bp;
5254         return 0;
5255 }
5256
5257 static void __net_exit ipv6_inetpeer_exit(struct net *net)
5258 {
5259         struct inet_peer_base *bp = net->ipv6.peers;
5260
5261         net->ipv6.peers = NULL;
5262         inetpeer_invalidate_tree(bp);
5263         kfree(bp);
5264 }
5265
5266 static struct pernet_operations ipv6_inetpeer_ops = {
5267         .init   =       ipv6_inetpeer_init,
5268         .exit   =       ipv6_inetpeer_exit,
5269 };
5270
5271 static struct pernet_operations ip6_route_net_late_ops = {
5272         .init = ip6_route_net_init_late,
5273         .exit = ip6_route_net_exit_late,
5274 };
5275
5276 static struct notifier_block ip6_route_dev_notifier = {
5277         .notifier_call = ip6_route_dev_notify,
5278         .priority = ADDRCONF_NOTIFY_PRIORITY - 10,
5279 };
5280
5281 void __init ip6_route_init_special_entries(void)
5282 {
5283         /* Registering of the loopback is done before this portion of code,
5284          * the loopback reference in rt6_info will not be taken, do it
5285          * manually for init_net */
5286         init_net.ipv6.fib6_null_entry->fib6_nh.nh_dev = init_net.loopback_dev;
5287         init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev;
5288         init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
5289   #ifdef CONFIG_IPV6_MULTIPLE_TABLES
5290         init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev;
5291         init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
5292         init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev;
5293         init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
5294   #endif
5295 }
5296
5297 int __init ip6_route_init(void)
5298 {
5299         int ret;
5300         int cpu;
5301
5302         ret = -ENOMEM;
5303         ip6_dst_ops_template.kmem_cachep =
5304                 kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0,
5305                                   SLAB_HWCACHE_ALIGN, NULL);
5306         if (!ip6_dst_ops_template.kmem_cachep)
5307                 goto out;
5308
5309         ret = dst_entries_init(&ip6_dst_blackhole_ops);
5310         if (ret)
5311                 goto out_kmem_cache;
5312
5313         ret = register_pernet_subsys(&ipv6_inetpeer_ops);
5314         if (ret)
5315                 goto out_dst_entries;
5316
5317         ret = register_pernet_subsys(&ip6_route_net_ops);
5318         if (ret)
5319                 goto out_register_inetpeer;
5320
5321         ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep;
5322
5323         ret = fib6_init();
5324         if (ret)
5325                 goto out_register_subsys;
5326
5327         ret = xfrm6_init();
5328         if (ret)
5329                 goto out_fib6_init;
5330
5331         ret = fib6_rules_init();
5332         if (ret)
5333                 goto xfrm6_init;
5334
5335         ret = register_pernet_subsys(&ip6_route_net_late_ops);
5336         if (ret)
5337                 goto fib6_rules_init;
5338
5339         ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_NEWROUTE,
5340                                    inet6_rtm_newroute, NULL, 0);
5341         if (ret < 0)
5342                 goto out_register_late_subsys;
5343
5344         ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_DELROUTE,
5345                                    inet6_rtm_delroute, NULL, 0);
5346         if (ret < 0)
5347                 goto out_register_late_subsys;
5348
5349         ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_GETROUTE,
5350                                    inet6_rtm_getroute, NULL,
5351                                    RTNL_FLAG_DOIT_UNLOCKED);
5352         if (ret < 0)
5353                 goto out_register_late_subsys;
5354
5355         ret = register_netdevice_notifier(&ip6_route_dev_notifier);
5356         if (ret)
5357                 goto out_register_late_subsys;
5358
5359         for_each_possible_cpu(cpu) {
5360                 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
5361
5362                 INIT_LIST_HEAD(&ul->head);
5363                 spin_lock_init(&ul->lock);
5364         }
5365
5366 out:
5367         return ret;
5368
5369 out_register_late_subsys:
5370         rtnl_unregister_all(PF_INET6);
5371         unregister_pernet_subsys(&ip6_route_net_late_ops);
5372 fib6_rules_init:
5373         fib6_rules_cleanup();
5374 xfrm6_init:
5375         xfrm6_fini();
5376 out_fib6_init:
5377         fib6_gc_cleanup();
5378 out_register_subsys:
5379         unregister_pernet_subsys(&ip6_route_net_ops);
5380 out_register_inetpeer:
5381         unregister_pernet_subsys(&ipv6_inetpeer_ops);
5382 out_dst_entries:
5383         dst_entries_destroy(&ip6_dst_blackhole_ops);
5384 out_kmem_cache:
5385         kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep);
5386         goto out;
5387 }
5388
5389 void ip6_route_cleanup(void)
5390 {
5391         unregister_netdevice_notifier(&ip6_route_dev_notifier);
5392         unregister_pernet_subsys(&ip6_route_net_late_ops);
5393         fib6_rules_cleanup();
5394         xfrm6_fini();
5395         fib6_gc_cleanup();
5396         unregister_pernet_subsys(&ipv6_inetpeer_ops);
5397         unregister_pernet_subsys(&ip6_route_net_ops);
5398         dst_entries_destroy(&ip6_dst_blackhole_ops);
5399         kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep);
5400 }