net/ipv4/nexthop.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Generic nexthop implementation
   3  *
   4  * Copyright (c) 2017-19 Cumulus Networks
   5  * Copyright (c) 2017-19 David Ahern <dsa@cumulusnetworks.com>
   6  */
   7
   8 #include <linux/nexthop.h>
   9 #include <linux/rtnetlink.h>
  10 #include <linux/slab.h>
  11 #include <net/arp.h>
  12 #include <net/ipv6_stubs.h>
  13 #include <net/lwtunnel.h>
  14 #include <net/ndisc.h>
  15 #include <net/nexthop.h>
  16 #include <net/route.h>
  17 #include <net/sock.h>
  18
  19 static void remove_nexthop(struct net *net, struct nexthop *nh,
  20                            struct nl_info *nlinfo);
  21
  22 #define NH_DEV_HASHBITS  8
  23 #define NH_DEV_HASHSIZE (1U << NH_DEV_HASHBITS)
  24
  25 static const struct nla_policy rtm_nh_policy_new[] = {
  26         [NHA_ID]                = { .type = NLA_U32 },
  27         [NHA_GROUP]             = { .type = NLA_BINARY },
  28         [NHA_GROUP_TYPE]        = { .type = NLA_U16 },
  29         [NHA_BLACKHOLE]         = { .type = NLA_FLAG },
  30         [NHA_OIF]               = { .type = NLA_U32 },
  31         [NHA_GATEWAY]           = { .type = NLA_BINARY },
  32         [NHA_ENCAP_TYPE]        = { .type = NLA_U16 },
  33         [NHA_ENCAP]             = { .type = NLA_NESTED },
  34         [NHA_FDB]               = { .type = NLA_FLAG },
  35 };
  36
  37 static const struct nla_policy rtm_nh_policy_get[] = {
  38         [NHA_ID]                = { .type = NLA_U32 },
  39 };
  40
  41 static const struct nla_policy rtm_nh_policy_dump[] = {
  42         [NHA_OIF]               = { .type = NLA_U32 },
  43         [NHA_GROUPS]            = { .type = NLA_FLAG },
  44         [NHA_MASTER]            = { .type = NLA_U32 },
  45         [NHA_FDB]               = { .type = NLA_FLAG },
  46 };
  47
  48 static bool nexthop_notifiers_is_empty(struct net *net)
  49 {
  50         return !net->nexthop.notifier_chain.head;
  51 }
  52
  53 static void
  54 __nh_notifier_single_info_init(struct nh_notifier_single_info *nh_info,
  55                                const struct nexthop *nh)
  56 {
  57         struct nh_info *nhi = rtnl_dereference(nh->nh_info);
  58
  59         nh_info->dev = nhi->fib_nhc.nhc_dev;
  60         nh_info->gw_family = nhi->fib_nhc.nhc_gw_family;
  61         if (nh_info->gw_family == AF_INET)
  62                 nh_info->ipv4 = nhi->fib_nhc.nhc_gw.ipv4;
  63         else if (nh_info->gw_family == AF_INET6)
  64                 nh_info->ipv6 = nhi->fib_nhc.nhc_gw.ipv6;
  65
  66         nh_info->is_reject = nhi->reject_nh;
  67         nh_info->is_fdb = nhi->fdb_nh;
  68         nh_info->has_encap = !!nhi->fib_nhc.nhc_lwtstate;
  69 }
  70
  71 static int nh_notifier_single_info_init(struct nh_notifier_info *info,
  72                                         const struct nexthop *nh)
  73 {
  74         info->type = NH_NOTIFIER_INFO_TYPE_SINGLE;
  75         info->nh = kzalloc(sizeof(*info->nh), GFP_KERNEL);
  76         if (!info->nh)
  77                 return -ENOMEM;
  78
  79         __nh_notifier_single_info_init(info->nh, nh);
  80
  81         return 0;
  82 }
  83
  84 static void nh_notifier_single_info_fini(struct nh_notifier_info *info)
  85 {
  86         kfree(info->nh);
  87 }
  88
  89 static int nh_notifier_mp_info_init(struct nh_notifier_info *info,
  90                                     struct nh_group *nhg)
  91 {
  92         u16 num_nh = nhg->num_nh;
  93         int i;
  94
  95         info->type = NH_NOTIFIER_INFO_TYPE_GRP;
  96         info->nh_grp = kzalloc(struct_size(info->nh_grp, nh_entries, num_nh),
  97                                GFP_KERNEL);
  98         if (!info->nh_grp)
  99                 return -ENOMEM;
 100
 101         info->nh_grp->num_nh = num_nh;
 102         info->nh_grp->is_fdb = nhg->fdb_nh;
 103
 104         for (i = 0; i < num_nh; i++) {
 105                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 106
 107                 info->nh_grp->nh_entries[i].id = nhge->nh->id;
 108                 info->nh_grp->nh_entries[i].weight = nhge->weight;
 109                 __nh_notifier_single_info_init(&info->nh_grp->nh_entries[i].nh,
 110                                                nhge->nh);
 111         }
 112
 113         return 0;
 114 }
 115
 116 static int nh_notifier_grp_info_init(struct nh_notifier_info *info,
 117                                      const struct nexthop *nh)
 118 {
 119         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 120
 121         if (nhg->mpath)
 122                 return nh_notifier_mp_info_init(info, nhg);
 123         return -EINVAL;
 124 }
 125
 126 static void nh_notifier_grp_info_fini(struct nh_notifier_info *info,
 127                                       const struct nexthop *nh)
 128 {
 129         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 130
 131         if (nhg->mpath)
 132                 kfree(info->nh_grp);
 133 }
 134
 135 static int nh_notifier_info_init(struct nh_notifier_info *info,
 136                                  const struct nexthop *nh)
 137 {
 138         info->id = nh->id;
 139
 140         if (nh->is_group)
 141                 return nh_notifier_grp_info_init(info, nh);
 142         else
 143                 return nh_notifier_single_info_init(info, nh);
 144 }
 145
 146 static void nh_notifier_info_fini(struct nh_notifier_info *info,
 147                                   const struct nexthop *nh)
 148 {
 149         if (nh->is_group)
 150                 nh_notifier_grp_info_fini(info, nh);
 151         else
 152                 nh_notifier_single_info_fini(info);
 153 }
 154
 155 static int call_nexthop_notifiers(struct net *net,
 156                                   enum nexthop_event_type event_type,
 157                                   struct nexthop *nh,
 158                                   struct netlink_ext_ack *extack)
 159 {
 160         struct nh_notifier_info info = {
 161                 .net = net,
 162                 .extack = extack,
 163         };
 164         int err;
 165
 166         ASSERT_RTNL();
 167
 168         if (nexthop_notifiers_is_empty(net))
 169                 return 0;
 170
 171         err = nh_notifier_info_init(&info, nh);
 172         if (err) {
 173                 NL_SET_ERR_MSG(extack, "Failed to initialize nexthop notifier info");
 174                 return err;
 175         }
 176
 177         err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
 178                                            event_type, &info);
 179         nh_notifier_info_fini(&info, nh);
 180
 181         return notifier_to_errno(err);
 182 }
 183
 184 static int call_nexthop_notifier(struct notifier_block *nb, struct net *net,
 185                                  enum nexthop_event_type event_type,
 186                                  struct nexthop *nh,
 187                                  struct netlink_ext_ack *extack)
 188 {
 189         struct nh_notifier_info info = {
 190                 .net = net,
 191                 .extack = extack,
 192         };
 193         int err;
 194
 195         err = nh_notifier_info_init(&info, nh);
 196         if (err)
 197                 return err;
 198
 199         err = nb->notifier_call(nb, event_type, &info);
 200         nh_notifier_info_fini(&info, nh);
 201
 202         return notifier_to_errno(err);
 203 }
 204
 205 static unsigned int nh_dev_hashfn(unsigned int val)
 206 {
 207         unsigned int mask = NH_DEV_HASHSIZE - 1;
 208
 209         return (val ^
 210                 (val >> NH_DEV_HASHBITS) ^
 211                 (val >> (NH_DEV_HASHBITS * 2))) & mask;
 212 }
 213
 214 static void nexthop_devhash_add(struct net *net, struct nh_info *nhi)
 215 {
 216         struct net_device *dev = nhi->fib_nhc.nhc_dev;
 217         struct hlist_head *head;
 218         unsigned int hash;
 219
 220         WARN_ON(!dev);
 221
 222         hash = nh_dev_hashfn(dev->ifindex);
 223         head = &net->nexthop.devhash[hash];
 224         hlist_add_head(&nhi->dev_hash, head);
 225 }
 226
 227 static void nexthop_free_group(struct nexthop *nh)
 228 {
 229         struct nh_group *nhg;
 230         int i;
 231
 232         nhg = rcu_dereference_raw(nh->nh_grp);
 233         for (i = 0; i < nhg->num_nh; ++i) {
 234                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 235
 236                 WARN_ON(!list_empty(&nhge->nh_list));
 237                 nexthop_put(nhge->nh);
 238         }
 239
 240         WARN_ON(nhg->spare == nhg);
 241
 242         kfree(nhg->spare);
 243         kfree(nhg);
 244 }
 245
 246 static void nexthop_free_single(struct nexthop *nh)
 247 {
 248         struct nh_info *nhi;
 249
 250         nhi = rcu_dereference_raw(nh->nh_info);
 251         switch (nhi->family) {
 252         case AF_INET:
 253                 fib_nh_release(nh->net, &nhi->fib_nh);
 254                 break;
 255         case AF_INET6:
 256                 ipv6_stub->fib6_nh_release(&nhi->fib6_nh);
 257                 break;
 258         }
 259         kfree(nhi);
 260 }
 261
 262 void nexthop_free_rcu(struct rcu_head *head)
 263 {
 264         struct nexthop *nh = container_of(head, struct nexthop, rcu);
 265
 266         if (nh->is_group)
 267                 nexthop_free_group(nh);
 268         else
 269                 nexthop_free_single(nh);
 270
 271         kfree(nh);
 272 }
 273 EXPORT_SYMBOL_GPL(nexthop_free_rcu);
 274
 275 static struct nexthop *nexthop_alloc(void)
 276 {
 277         struct nexthop *nh;
 278
 279         nh = kzalloc(sizeof(struct nexthop), GFP_KERNEL);
 280         if (nh) {
 281                 INIT_LIST_HEAD(&nh->fi_list);
 282                 INIT_LIST_HEAD(&nh->f6i_list);
 283                 INIT_LIST_HEAD(&nh->grp_list);
 284                 INIT_LIST_HEAD(&nh->fdb_list);
 285         }
 286         return nh;
 287 }
 288
 289 static struct nh_group *nexthop_grp_alloc(u16 num_nh)
 290 {
 291         struct nh_group *nhg;
 292
 293         nhg = kzalloc(struct_size(nhg, nh_entries, num_nh), GFP_KERNEL);
 294         if (nhg)
 295                 nhg->num_nh = num_nh;
 296
 297         return nhg;
 298 }
 299
 300 static void nh_base_seq_inc(struct net *net)
 301 {
 302         while (++net->nexthop.seq == 0)
 303                 ;
 304 }
 305
 306 /* no reference taken; rcu lock or rtnl must be held */
 307 struct nexthop *nexthop_find_by_id(struct net *net, u32 id)
 308 {
 309         struct rb_node **pp, *parent = NULL, *next;
 310
 311         pp = &net->nexthop.rb_root.rb_node;
 312         while (1) {
 313                 struct nexthop *nh;
 314
 315                 next = rcu_dereference_raw(*pp);
 316                 if (!next)
 317                         break;
 318                 parent = next;
 319
 320                 nh = rb_entry(parent, struct nexthop, rb_node);
 321                 if (id < nh->id)
 322                         pp = &next->rb_left;
 323                 else if (id > nh->id)
 324                         pp = &next->rb_right;
 325                 else
 326                         return nh;
 327         }
 328         return NULL;
 329 }
 330 EXPORT_SYMBOL_GPL(nexthop_find_by_id);
 331
 332 /* used for auto id allocation; called with rtnl held */
 333 static u32 nh_find_unused_id(struct net *net)
 334 {
 335         u32 id_start = net->nexthop.last_id_allocated;
 336
 337         while (1) {
 338                 net->nexthop.last_id_allocated++;
 339                 if (net->nexthop.last_id_allocated == id_start)
 340                         break;
 341
 342                 if (!nexthop_find_by_id(net, net->nexthop.last_id_allocated))
 343                         return net->nexthop.last_id_allocated;
 344         }
 345         return 0;
 346 }
 347
 348 static int nla_put_nh_group(struct sk_buff *skb, struct nh_group *nhg)
 349 {
 350         struct nexthop_grp *p;
 351         size_t len = nhg->num_nh * sizeof(*p);
 352         struct nlattr *nla;
 353         u16 group_type = 0;
 354         int i;
 355
 356         if (nhg->mpath)
 357                 group_type = NEXTHOP_GRP_TYPE_MPATH;
 358
 359         if (nla_put_u16(skb, NHA_GROUP_TYPE, group_type))
 360                 goto nla_put_failure;
 361
 362         nla = nla_reserve(skb, NHA_GROUP, len);
 363         if (!nla)
 364                 goto nla_put_failure;
 365
 366         p = nla_data(nla);
 367         for (i = 0; i < nhg->num_nh; ++i) {
 368                 p->id = nhg->nh_entries[i].nh->id;
 369                 p->weight = nhg->nh_entries[i].weight - 1;
 370                 p += 1;
 371         }
 372
 373         return 0;
 374
 375 nla_put_failure:
 376         return -EMSGSIZE;
 377 }
 378
 379 static int nh_fill_node(struct sk_buff *skb, struct nexthop *nh,
 380                         int event, u32 portid, u32 seq, unsigned int nlflags)
 381 {
 382         struct fib6_nh *fib6_nh;
 383         struct fib_nh *fib_nh;
 384         struct nlmsghdr *nlh;
 385         struct nh_info *nhi;
 386         struct nhmsg *nhm;
 387
 388         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
 389         if (!nlh)
 390                 return -EMSGSIZE;
 391
 392         nhm = nlmsg_data(nlh);
 393         nhm->nh_family = AF_UNSPEC;
 394         nhm->nh_flags = nh->nh_flags;
 395         nhm->nh_protocol = nh->protocol;
 396         nhm->nh_scope = 0;
 397         nhm->resvd = 0;
 398
 399         if (nla_put_u32(skb, NHA_ID, nh->id))
 400                 goto nla_put_failure;
 401
 402         if (nh->is_group) {
 403                 struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 404
 405                 if (nhg->fdb_nh && nla_put_flag(skb, NHA_FDB))
 406                         goto nla_put_failure;
 407                 if (nla_put_nh_group(skb, nhg))
 408                         goto nla_put_failure;
 409                 goto out;
 410         }
 411
 412         nhi = rtnl_dereference(nh->nh_info);
 413         nhm->nh_family = nhi->family;
 414         if (nhi->reject_nh) {
 415                 if (nla_put_flag(skb, NHA_BLACKHOLE))
 416                         goto nla_put_failure;
 417                 goto out;
 418         } else if (nhi->fdb_nh) {
 419                 if (nla_put_flag(skb, NHA_FDB))
 420                         goto nla_put_failure;
 421         } else {
 422                 const struct net_device *dev;
 423
 424                 dev = nhi->fib_nhc.nhc_dev;
 425                 if (dev && nla_put_u32(skb, NHA_OIF, dev->ifindex))
 426                         goto nla_put_failure;
 427         }
 428
 429         nhm->nh_scope = nhi->fib_nhc.nhc_scope;
 430         switch (nhi->family) {
 431         case AF_INET:
 432                 fib_nh = &nhi->fib_nh;
 433                 if (fib_nh->fib_nh_gw_family &&
 434                     nla_put_be32(skb, NHA_GATEWAY, fib_nh->fib_nh_gw4))
 435                         goto nla_put_failure;
 436                 break;
 437
 438         case AF_INET6:
 439                 fib6_nh = &nhi->fib6_nh;
 440                 if (fib6_nh->fib_nh_gw_family &&
 441                     nla_put_in6_addr(skb, NHA_GATEWAY, &fib6_nh->fib_nh_gw6))
 442                         goto nla_put_failure;
 443                 break;
 444         }
 445
 446         if (nhi->fib_nhc.nhc_lwtstate &&
 447             lwtunnel_fill_encap(skb, nhi->fib_nhc.nhc_lwtstate,
 448                                 NHA_ENCAP, NHA_ENCAP_TYPE) < 0)
 449                 goto nla_put_failure;
 450
 451 out:
 452         nlmsg_end(skb, nlh);
 453         return 0;
 454
 455 nla_put_failure:
 456         nlmsg_cancel(skb, nlh);
 457         return -EMSGSIZE;
 458 }
 459
 460 static size_t nh_nlmsg_size_grp(struct nexthop *nh)
 461 {
 462         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 463         size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh;
 464
 465         return nla_total_size(sz) +
 466                nla_total_size(2);  /* NHA_GROUP_TYPE */
 467 }
 468
 469 static size_t nh_nlmsg_size_single(struct nexthop *nh)
 470 {
 471         struct nh_info *nhi = rtnl_dereference(nh->nh_info);
 472         size_t sz;
 473
 474         /* covers NHA_BLACKHOLE since NHA_OIF and BLACKHOLE
 475          * are mutually exclusive
 476          */
 477         sz = nla_total_size(4);  /* NHA_OIF */
 478
 479         switch (nhi->family) {
 480         case AF_INET:
 481                 if (nhi->fib_nh.fib_nh_gw_family)
 482                         sz += nla_total_size(4);  /* NHA_GATEWAY */
 483                 break;
 484
 485         case AF_INET6:
 486                 /* NHA_GATEWAY */
 487                 if (nhi->fib6_nh.fib_nh_gw_family)
 488                         sz += nla_total_size(sizeof(const struct in6_addr));
 489                 break;
 490         }
 491
 492         if (nhi->fib_nhc.nhc_lwtstate) {
 493                 sz += lwtunnel_get_encap_size(nhi->fib_nhc.nhc_lwtstate);
 494                 sz += nla_total_size(2);  /* NHA_ENCAP_TYPE */
 495         }
 496
 497         return sz;
 498 }
 499
 500 static size_t nh_nlmsg_size(struct nexthop *nh)
 501 {
 502         size_t sz = NLMSG_ALIGN(sizeof(struct nhmsg));
 503
 504         sz += nla_total_size(4); /* NHA_ID */
 505
 506         if (nh->is_group)
 507                 sz += nh_nlmsg_size_grp(nh);
 508         else
 509                 sz += nh_nlmsg_size_single(nh);
 510
 511         return sz;
 512 }
 513
 514 static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info)
 515 {
 516         unsigned int nlflags = info->nlh ? info->nlh->nlmsg_flags : 0;
 517         u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
 518         struct sk_buff *skb;
 519         int err = -ENOBUFS;
 520
 521         skb = nlmsg_new(nh_nlmsg_size(nh), gfp_any());
 522         if (!skb)
 523                 goto errout;
 524
 525         err = nh_fill_node(skb, nh, event, info->portid, seq, nlflags);
 526         if (err < 0) {
 527                 /* -EMSGSIZE implies BUG in nh_nlmsg_size() */
 528                 WARN_ON(err == -EMSGSIZE);
 529                 kfree_skb(skb);
 530                 goto errout;
 531         }
 532
 533         rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_NEXTHOP,
 534                     info->nlh, gfp_any());
 535         return;
 536 errout:
 537         if (err < 0)
 538                 rtnl_set_sk_err(info->nl_net, RTNLGRP_NEXTHOP, err);
 539 }
 540
 541 static bool valid_group_nh(struct nexthop *nh, unsigned int npaths,
 542                            bool *is_fdb, struct netlink_ext_ack *extack)
 543 {
 544         if (nh->is_group) {
 545                 struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 546
 547                 /* nested multipath (group within a group) is not
 548                  * supported
 549                  */
 550                 if (nhg->mpath) {
 551                         NL_SET_ERR_MSG(extack,
 552                                        "Multipath group can not be a nexthop within a group");
 553                         return false;
 554                 }
 555                 *is_fdb = nhg->fdb_nh;
 556         } else {
 557                 struct nh_info *nhi = rtnl_dereference(nh->nh_info);
 558
 559                 if (nhi->reject_nh && npaths > 1) {
 560                         NL_SET_ERR_MSG(extack,
 561                                        "Blackhole nexthop can not be used in a group with more than 1 path");
 562                         return false;
 563                 }
 564                 *is_fdb = nhi->fdb_nh;
 565         }
 566
 567         return true;
 568 }
 569
 570 static int nh_check_attr_fdb_group(struct nexthop *nh, u8 *nh_family,
 571                                    struct netlink_ext_ack *extack)
 572 {
 573         struct nh_info *nhi;
 574
 575         nhi = rtnl_dereference(nh->nh_info);
 576
 577         if (!nhi->fdb_nh) {
 578                 NL_SET_ERR_MSG(extack, "FDB nexthop group can only have fdb nexthops");
 579                 return -EINVAL;
 580         }
 581
 582         if (*nh_family == AF_UNSPEC) {
 583                 *nh_family = nhi->family;
 584         } else if (*nh_family != nhi->family) {
 585                 NL_SET_ERR_MSG(extack, "FDB nexthop group cannot have mixed family nexthops");
 586                 return -EINVAL;
 587         }
 588
 589         return 0;
 590 }
 591
 592 static int nh_check_attr_group(struct net *net,
 593                                struct nlattr *tb[], size_t tb_size,
 594                                struct netlink_ext_ack *extack)
 595 {
 596         unsigned int len = nla_len(tb[NHA_GROUP]);
 597         u8 nh_family = AF_UNSPEC;
 598         struct nexthop_grp *nhg;
 599         unsigned int i, j;
 600         u8 nhg_fdb = 0;
 601
 602         if (!len || len & (sizeof(struct nexthop_grp) - 1)) {
 603                 NL_SET_ERR_MSG(extack,
 604                                "Invalid length for nexthop group attribute");
 605                 return -EINVAL;
 606         }
 607
 608         /* convert len to number of nexthop ids */
 609         len /= sizeof(*nhg);
 610
 611         nhg = nla_data(tb[NHA_GROUP]);
 612         for (i = 0; i < len; ++i) {
 613                 if (nhg[i].resvd1 || nhg[i].resvd2) {
 614                         NL_SET_ERR_MSG(extack, "Reserved fields in nexthop_grp must be 0");
 615                         return -EINVAL;
 616                 }
 617                 if (nhg[i].weight > 254) {
 618                         NL_SET_ERR_MSG(extack, "Invalid value for weight");
 619                         return -EINVAL;
 620                 }
 621                 for (j = i + 1; j < len; ++j) {
 622                         if (nhg[i].id == nhg[j].id) {
 623                                 NL_SET_ERR_MSG(extack, "Nexthop id can not be used twice in a group");
 624                                 return -EINVAL;
 625                         }
 626                 }
 627         }
 628
 629         if (tb[NHA_FDB])
 630                 nhg_fdb = 1;
 631         nhg = nla_data(tb[NHA_GROUP]);
 632         for (i = 0; i < len; ++i) {
 633                 struct nexthop *nh;
 634                 bool is_fdb_nh;
 635
 636                 nh = nexthop_find_by_id(net, nhg[i].id);
 637                 if (!nh) {
 638                         NL_SET_ERR_MSG(extack, "Invalid nexthop id");
 639                         return -EINVAL;
 640                 }
 641                 if (!valid_group_nh(nh, len, &is_fdb_nh, extack))
 642                         return -EINVAL;
 643
 644                 if (nhg_fdb && nh_check_attr_fdb_group(nh, &nh_family, extack))
 645                         return -EINVAL;
 646
 647                 if (!nhg_fdb && is_fdb_nh) {
 648                         NL_SET_ERR_MSG(extack, "Non FDB nexthop group cannot have fdb nexthops");
 649                         return -EINVAL;
 650                 }
 651         }
 652         for (i = NHA_GROUP_TYPE + 1; i < tb_size; ++i) {
 653                 if (!tb[i])
 654                         continue;
 655                 if (i == NHA_FDB)
 656                         continue;
 657                 NL_SET_ERR_MSG(extack,
 658                                "No other attributes can be set in nexthop groups");
 659                 return -EINVAL;
 660         }
 661
 662         return 0;
 663 }
 664
 665 static bool ipv6_good_nh(const struct fib6_nh *nh)
 666 {
 667         int state = NUD_REACHABLE;
 668         struct neighbour *n;
 669
 670         rcu_read_lock_bh();
 671
 672         n = __ipv6_neigh_lookup_noref_stub(nh->fib_nh_dev, &nh->fib_nh_gw6);
 673         if (n)
 674                 state = n->nud_state;
 675
 676         rcu_read_unlock_bh();
 677
 678         return !!(state & NUD_VALID);
 679 }
 680
 681 static bool ipv4_good_nh(const struct fib_nh *nh)
 682 {
 683         int state = NUD_REACHABLE;
 684         struct neighbour *n;
 685
 686         rcu_read_lock_bh();
 687
 688         n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
 689                                       (__force u32)nh->fib_nh_gw4);
 690         if (n)
 691                 state = n->nud_state;
 692
 693         rcu_read_unlock_bh();
 694
 695         return !!(state & NUD_VALID);
 696 }
 697
 698 static struct nexthop *nexthop_select_path_mp(struct nh_group *nhg, int hash)
 699 {
 700         struct nexthop *rc = NULL;
 701         int i;
 702
 703         for (i = 0; i < nhg->num_nh; ++i) {
 704                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 705                 struct nh_info *nhi;
 706
 707                 if (hash > atomic_read(&nhge->mpath.upper_bound))
 708                         continue;
 709
 710                 nhi = rcu_dereference(nhge->nh->nh_info);
 711                 if (nhi->fdb_nh)
 712                         return nhge->nh;
 713
 714                 /* nexthops always check if it is good and does
 715                  * not rely on a sysctl for this behavior
 716                  */
 717                 switch (nhi->family) {
 718                 case AF_INET:
 719                         if (ipv4_good_nh(&nhi->fib_nh))
 720                                 return nhge->nh;
 721                         break;
 722                 case AF_INET6:
 723                         if (ipv6_good_nh(&nhi->fib6_nh))
 724                                 return nhge->nh;
 725                         break;
 726                 }
 727
 728                 if (!rc)
 729                         rc = nhge->nh;
 730         }
 731
 732         return rc;
 733 }
 734
 735 struct nexthop *nexthop_select_path(struct nexthop *nh, int hash)
 736 {
 737         struct nh_group *nhg;
 738
 739         if (!nh->is_group)
 740                 return nh;
 741
 742         nhg = rcu_dereference(nh->nh_grp);
 743         if (nhg->mpath)
 744                 return nexthop_select_path_mp(nhg, hash);
 745
 746         /* Unreachable. */
 747         return NULL;
 748 }
 749 EXPORT_SYMBOL_GPL(nexthop_select_path);
 750
 751 int nexthop_for_each_fib6_nh(struct nexthop *nh,
 752                              int (*cb)(struct fib6_nh *nh, void *arg),
 753                              void *arg)
 754 {
 755         struct nh_info *nhi;
 756         int err;
 757
 758         if (nh->is_group) {
 759                 struct nh_group *nhg;
 760                 int i;
 761
 762                 nhg = rcu_dereference_rtnl(nh->nh_grp);
 763                 for (i = 0; i < nhg->num_nh; i++) {
 764                         struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 765
 766                         nhi = rcu_dereference_rtnl(nhge->nh->nh_info);
 767                         err = cb(&nhi->fib6_nh, arg);
 768                         if (err)
 769                                 return err;
 770                 }
 771         } else {
 772                 nhi = rcu_dereference_rtnl(nh->nh_info);
 773                 err = cb(&nhi->fib6_nh, arg);
 774                 if (err)
 775                         return err;
 776         }
 777
 778         return 0;
 779 }
 780 EXPORT_SYMBOL_GPL(nexthop_for_each_fib6_nh);
 781
 782 static int check_src_addr(const struct in6_addr *saddr,
 783                           struct netlink_ext_ack *extack)
 784 {
 785         if (!ipv6_addr_any(saddr)) {
 786                 NL_SET_ERR_MSG(extack, "IPv6 routes using source address can not use nexthop objects");
 787                 return -EINVAL;
 788         }
 789         return 0;
 790 }
 791
 792 int fib6_check_nexthop(struct nexthop *nh, struct fib6_config *cfg,
 793                        struct netlink_ext_ack *extack)
 794 {
 795         struct nh_info *nhi;
 796         bool is_fdb_nh;
 797
 798         /* fib6_src is unique to a fib6_info and limits the ability to cache
 799          * routes in fib6_nh within a nexthop that is potentially shared
 800          * across multiple fib entries. If the config wants to use source
 801          * routing it can not use nexthop objects. mlxsw also does not allow
 802          * fib6_src on routes.
 803          */
 804         if (cfg && check_src_addr(&cfg->fc_src, extack) < 0)
 805                 return -EINVAL;
 806
 807         if (nh->is_group) {
 808                 struct nh_group *nhg;
 809
 810                 nhg = rtnl_dereference(nh->nh_grp);
 811                 if (nhg->has_v4)
 812                         goto no_v4_nh;
 813                 is_fdb_nh = nhg->fdb_nh;
 814         } else {
 815                 nhi = rtnl_dereference(nh->nh_info);
 816                 if (nhi->family == AF_INET)
 817                         goto no_v4_nh;
 818                 is_fdb_nh = nhi->fdb_nh;
 819         }
 820
 821         if (is_fdb_nh) {
 822                 NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
 823                 return -EINVAL;
 824         }
 825
 826         return 0;
 827 no_v4_nh:
 828         NL_SET_ERR_MSG(extack, "IPv6 routes can not use an IPv4 nexthop");
 829         return -EINVAL;
 830 }
 831 EXPORT_SYMBOL_GPL(fib6_check_nexthop);
 832
 833 /* if existing nexthop has ipv6 routes linked to it, need
 834  * to verify this new spec works with ipv6
 835  */
 836 static int fib6_check_nh_list(struct nexthop *old, struct nexthop *new,
 837                               struct netlink_ext_ack *extack)
 838 {
 839         struct fib6_info *f6i;
 840
 841         if (list_empty(&old->f6i_list))
 842                 return 0;
 843
 844         list_for_each_entry(f6i, &old->f6i_list, nh_list) {
 845                 if (check_src_addr(&f6i->fib6_src.addr, extack) < 0)
 846                         return -EINVAL;
 847         }
 848
 849         return fib6_check_nexthop(new, NULL, extack);
 850 }
 851
 852 static int nexthop_check_scope(struct nh_info *nhi, u8 scope,
 853                                struct netlink_ext_ack *extack)
 854 {
 855         if (scope == RT_SCOPE_HOST && nhi->fib_nhc.nhc_gw_family) {
 856                 NL_SET_ERR_MSG(extack,
 857                                "Route with host scope can not have a gateway");
 858                 return -EINVAL;
 859         }
 860
 861         if (nhi->fib_nhc.nhc_flags & RTNH_F_ONLINK && scope >= RT_SCOPE_LINK) {
 862                 NL_SET_ERR_MSG(extack, "Scope mismatch with nexthop");
 863                 return -EINVAL;
 864         }
 865
 866         return 0;
 867 }
 868
 869 /* Invoked by fib add code to verify nexthop by id is ok with
 870  * config for prefix; parts of fib_check_nh not done when nexthop
 871  * object is used.
 872  */
 873 int fib_check_nexthop(struct nexthop *nh, u8 scope,
 874                       struct netlink_ext_ack *extack)
 875 {
 876         struct nh_info *nhi;
 877         int err = 0;
 878
 879         if (nh->is_group) {
 880                 struct nh_group *nhg;
 881
 882                 nhg = rtnl_dereference(nh->nh_grp);
 883                 if (nhg->fdb_nh) {
 884                         NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
 885                         err = -EINVAL;
 886                         goto out;
 887                 }
 888
 889                 if (scope == RT_SCOPE_HOST) {
 890                         NL_SET_ERR_MSG(extack, "Route with host scope can not have multiple nexthops");
 891                         err = -EINVAL;
 892                         goto out;
 893                 }
 894
 895                 /* all nexthops in a group have the same scope */
 896                 nhi = rtnl_dereference(nhg->nh_entries[0].nh->nh_info);
 897                 err = nexthop_check_scope(nhi, scope, extack);
 898         } else {
 899                 nhi = rtnl_dereference(nh->nh_info);
 900                 if (nhi->fdb_nh) {
 901                         NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
 902                         err = -EINVAL;
 903                         goto out;
 904                 }
 905                 err = nexthop_check_scope(nhi, scope, extack);
 906         }
 907
 908 out:
 909         return err;
 910 }
 911
 912 static int fib_check_nh_list(struct nexthop *old, struct nexthop *new,
 913                              struct netlink_ext_ack *extack)
 914 {
 915         struct fib_info *fi;
 916
 917         list_for_each_entry(fi, &old->fi_list, nh_list) {
 918                 int err;
 919
 920                 err = fib_check_nexthop(new, fi->fib_scope, extack);
 921                 if (err)
 922                         return err;
 923         }
 924         return 0;
 925 }
 926
 927 static void nh_group_rebalance(struct nh_group *nhg)
 928 {
 929         int total = 0;
 930         int w = 0;
 931         int i;
 932
 933         for (i = 0; i < nhg->num_nh; ++i)
 934                 total += nhg->nh_entries[i].weight;
 935
 936         for (i = 0; i < nhg->num_nh; ++i) {
 937                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 938                 int upper_bound;
 939
 940                 w += nhge->weight;
 941                 upper_bound = DIV_ROUND_CLOSEST_ULL((u64)w << 31, total) - 1;
 942                 atomic_set(&nhge->mpath.upper_bound, upper_bound);
 943         }
 944 }
 945
 946 static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
 947                                 struct nl_info *nlinfo)
 948 {
 949         struct nh_grp_entry *nhges, *new_nhges;
 950         struct nexthop *nhp = nhge->nh_parent;
 951         struct netlink_ext_ack extack;
 952         struct nexthop *nh = nhge->nh;
 953         struct nh_group *nhg, *newg;
 954         int i, j, err;
 955
 956         WARN_ON(!nh);
 957
 958         nhg = rtnl_dereference(nhp->nh_grp);
 959         newg = nhg->spare;
 960
 961         /* last entry, keep it visible and remove the parent */
 962         if (nhg->num_nh == 1) {
 963                 remove_nexthop(net, nhp, nlinfo);
 964                 return;
 965         }
 966
 967         newg->has_v4 = false;
 968         newg->mpath = nhg->mpath;
 969         newg->fdb_nh = nhg->fdb_nh;
 970         newg->num_nh = nhg->num_nh;
 971
 972         /* copy old entries to new except the one getting removed */
 973         nhges = nhg->nh_entries;
 974         new_nhges = newg->nh_entries;
 975         for (i = 0, j = 0; i < nhg->num_nh; ++i) {
 976                 struct nh_info *nhi;
 977
 978                 /* current nexthop getting removed */
 979                 if (nhg->nh_entries[i].nh == nh) {
 980                         newg->num_nh--;
 981                         continue;
 982                 }
 983
 984                 nhi = rtnl_dereference(nhges[i].nh->nh_info);
 985                 if (nhi->family == AF_INET)
 986                         newg->has_v4 = true;
 987
 988                 list_del(&nhges[i].nh_list);
 989                 new_nhges[j].nh_parent = nhges[i].nh_parent;
 990                 new_nhges[j].nh = nhges[i].nh;
 991                 new_nhges[j].weight = nhges[i].weight;
 992                 list_add(&new_nhges[j].nh_list, &new_nhges[j].nh->grp_list);
 993                 j++;
 994         }
 995
 996         nh_group_rebalance(newg);
 997         rcu_assign_pointer(nhp->nh_grp, newg);
 998
 999         list_del(&nhge->nh_list);
1000         nexthop_put(nhge->nh);
1001
1002         err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, nhp, &extack);
1003         if (err)
1004                 pr_err("%s\n", extack._msg);
1005
1006         if (nlinfo)
1007                 nexthop_notify(RTM_NEWNEXTHOP, nhp, nlinfo);
1008 }
1009
1010 static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
1011                                        struct nl_info *nlinfo)
1012 {
1013         struct nh_grp_entry *nhge, *tmp;
1014
1015         list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list)
1016                 remove_nh_grp_entry(net, nhge, nlinfo);
1017
1018         /* make sure all see the newly published array before releasing rtnl */
1019         synchronize_net();
1020 }
1021
1022 static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)
1023 {
1024         struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);
1025         int i, num_nh = nhg->num_nh;
1026
1027         for (i = 0; i < num_nh; ++i) {
1028                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1029
1030                 if (WARN_ON(!nhge->nh))
1031                         continue;
1032
1033                 list_del_init(&nhge->nh_list);
1034         }
1035 }
1036
1037 /* not called for nexthop replace */
1038 static void __remove_nexthop_fib(struct net *net, struct nexthop *nh)
1039 {
1040         struct fib6_info *f6i, *tmp;
1041         bool do_flush = false;
1042         struct fib_info *fi;
1043
1044         list_for_each_entry(fi, &nh->fi_list, nh_list) {
1045                 fi->fib_flags |= RTNH_F_DEAD;
1046                 do_flush = true;
1047         }
1048         if (do_flush)
1049                 fib_flush(net);
1050
1051         /* ip6_del_rt removes the entry from this list hence the _safe */
1052         list_for_each_entry_safe(f6i, tmp, &nh->f6i_list, nh_list) {
1053                 /* __ip6_del_rt does a release, so do a hold here */
1054                 fib6_info_hold(f6i);
1055                 ipv6_stub->ip6_del_rt(net, f6i,
1056                                       !net->ipv4.sysctl_nexthop_compat_mode);
1057         }
1058 }
1059
1060 static void __remove_nexthop(struct net *net, struct nexthop *nh,
1061                              struct nl_info *nlinfo)
1062 {
1063         __remove_nexthop_fib(net, nh);
1064
1065         if (nh->is_group) {
1066                 remove_nexthop_group(nh, nlinfo);
1067         } else {
1068                 struct nh_info *nhi;
1069
1070                 nhi = rtnl_dereference(nh->nh_info);
1071                 if (nhi->fib_nhc.nhc_dev)
1072                         hlist_del(&nhi->dev_hash);
1073
1074                 remove_nexthop_from_groups(net, nh, nlinfo);
1075         }
1076 }
1077
1078 static void remove_nexthop(struct net *net, struct nexthop *nh,
1079                            struct nl_info *nlinfo)
1080 {
1081         call_nexthop_notifiers(net, NEXTHOP_EVENT_DEL, nh, NULL);
1082
1083         /* remove from the tree */
1084         rb_erase(&nh->rb_node, &net->nexthop.rb_root);
1085
1086         if (nlinfo)
1087                 nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo);
1088
1089         __remove_nexthop(net, nh, nlinfo);
1090         nh_base_seq_inc(net);
1091
1092         nexthop_put(nh);
1093 }
1094
1095 /* if any FIB entries reference this nexthop, any dst entries
1096  * need to be regenerated
1097  */
1098 static void nh_rt_cache_flush(struct net *net, struct nexthop *nh)
1099 {
1100         struct fib6_info *f6i;
1101
1102         if (!list_empty(&nh->fi_list))
1103                 rt_cache_flush(net);
1104
1105         list_for_each_entry(f6i, &nh->f6i_list, nh_list)
1106                 ipv6_stub->fib6_update_sernum(net, f6i);
1107 }
1108
1109 static int replace_nexthop_grp(struct net *net, struct nexthop *old,
1110                                struct nexthop *new,
1111                                struct netlink_ext_ack *extack)
1112 {
1113         struct nh_group *oldg, *newg;
1114         int i, err;
1115
1116         if (!new->is_group) {
1117                 NL_SET_ERR_MSG(extack, "Can not replace a nexthop group with a nexthop.");
1118                 return -EINVAL;
1119         }
1120
1121         err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new, extack);
1122         if (err)
1123                 return err;
1124
1125         oldg = rtnl_dereference(old->nh_grp);
1126         newg = rtnl_dereference(new->nh_grp);
1127
1128         /* update parents - used by nexthop code for cleanup */
1129         for (i = 0; i < newg->num_nh; i++)
1130                 newg->nh_entries[i].nh_parent = old;
1131
1132         rcu_assign_pointer(old->nh_grp, newg);
1133
1134         for (i = 0; i < oldg->num_nh; i++)
1135                 oldg->nh_entries[i].nh_parent = new;
1136
1137         rcu_assign_pointer(new->nh_grp, oldg);
1138
1139         return 0;
1140 }
1141
1142 static void nh_group_v4_update(struct nh_group *nhg)
1143 {
1144         struct nh_grp_entry *nhges;
1145         bool has_v4 = false;
1146         int i;
1147
1148         nhges = nhg->nh_entries;
1149         for (i = 0; i < nhg->num_nh; i++) {
1150                 struct nh_info *nhi;
1151
1152                 nhi = rtnl_dereference(nhges[i].nh->nh_info);
1153                 if (nhi->family == AF_INET)
1154                         has_v4 = true;
1155         }
1156         nhg->has_v4 = has_v4;
1157 }
1158
1159 static int replace_nexthop_single(struct net *net, struct nexthop *old,
1160                                   struct nexthop *new,
1161                                   struct netlink_ext_ack *extack)
1162 {
1163         u8 old_protocol, old_nh_flags;
1164         struct nh_info *oldi, *newi;
1165         struct nh_grp_entry *nhge;
1166         int err;
1167
1168         if (new->is_group) {
1169                 NL_SET_ERR_MSG(extack, "Can not replace a nexthop with a nexthop group.");
1170                 return -EINVAL;
1171         }
1172
1173         err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new, extack);
1174         if (err)
1175                 return err;
1176
1177         /* Hardware flags were set on 'old' as 'new' is not in the red-black
1178          * tree. Therefore, inherit the flags from 'old' to 'new'.
1179          */
1180         new->nh_flags |= old->nh_flags & (RTNH_F_OFFLOAD | RTNH_F_TRAP);
1181
1182         oldi = rtnl_dereference(old->nh_info);
1183         newi = rtnl_dereference(new->nh_info);
1184
1185         newi->nh_parent = old;
1186         oldi->nh_parent = new;
1187
1188         old_protocol = old->protocol;
1189         old_nh_flags = old->nh_flags;
1190
1191         old->protocol = new->protocol;
1192         old->nh_flags = new->nh_flags;
1193
1194         rcu_assign_pointer(old->nh_info, newi);
1195         rcu_assign_pointer(new->nh_info, oldi);
1196
1197         /* Send a replace notification for all the groups using the nexthop. */
1198         list_for_each_entry(nhge, &old->grp_list, nh_list) {
1199                 struct nexthop *nhp = nhge->nh_parent;
1200
1201                 err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, nhp,
1202                                              extack);
1203                 if (err)
1204                         goto err_notify;
1205         }
1206
1207         /* When replacing an IPv4 nexthop with an IPv6 nexthop, potentially
1208          * update IPv4 indication in all the groups using the nexthop.
1209          */
1210         if (oldi->family == AF_INET && newi->family == AF_INET6) {
1211                 list_for_each_entry(nhge, &old->grp_list, nh_list) {
1212                         struct nexthop *nhp = nhge->nh_parent;
1213                         struct nh_group *nhg;
1214
1215                         nhg = rtnl_dereference(nhp->nh_grp);
1216                         nh_group_v4_update(nhg);
1217                 }
1218         }
1219
1220         return 0;
1221
1222 err_notify:
1223         rcu_assign_pointer(new->nh_info, newi);
1224         rcu_assign_pointer(old->nh_info, oldi);
1225         old->nh_flags = old_nh_flags;
1226         old->protocol = old_protocol;
1227         oldi->nh_parent = old;
1228         newi->nh_parent = new;
1229         list_for_each_entry_continue_reverse(nhge, &old->grp_list, nh_list) {
1230                 struct nexthop *nhp = nhge->nh_parent;
1231
1232                 call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, nhp, extack);
1233         }
1234         call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, old, extack);
1235         return err;
1236 }
1237
1238 static void __nexthop_replace_notify(struct net *net, struct nexthop *nh,
1239                                      struct nl_info *info)
1240 {
1241         struct fib6_info *f6i;
1242
1243         if (!list_empty(&nh->fi_list)) {
1244                 struct fib_info *fi;
1245
1246                 /* expectation is a few fib_info per nexthop and then
1247                  * a lot of routes per fib_info. So mark the fib_info
1248                  * and then walk the fib tables once
1249                  */
1250                 list_for_each_entry(fi, &nh->fi_list, nh_list)
1251                         fi->nh_updated = true;
1252
1253                 fib_info_notify_update(net, info);
1254
1255                 list_for_each_entry(fi, &nh->fi_list, nh_list)
1256                         fi->nh_updated = false;
1257         }
1258
1259         list_for_each_entry(f6i, &nh->f6i_list, nh_list)
1260                 ipv6_stub->fib6_rt_update(net, f6i, info);
1261 }
1262
1263 /* send RTM_NEWROUTE with REPLACE flag set for all FIB entries
1264  * linked to this nexthop and for all groups that the nexthop
1265  * is a member of
1266  */
1267 static void nexthop_replace_notify(struct net *net, struct nexthop *nh,
1268                                    struct nl_info *info)
1269 {
1270         struct nh_grp_entry *nhge;
1271
1272         __nexthop_replace_notify(net, nh, info);
1273
1274         list_for_each_entry(nhge, &nh->grp_list, nh_list)
1275                 __nexthop_replace_notify(net, nhge->nh_parent, info);
1276 }
1277
1278 static int replace_nexthop(struct net *net, struct nexthop *old,
1279                            struct nexthop *new, struct netlink_ext_ack *extack)
1280 {
1281         bool new_is_reject = false;
1282         struct nh_grp_entry *nhge;
1283         int err;
1284
1285         /* check that existing FIB entries are ok with the
1286          * new nexthop definition
1287          */
1288         err = fib_check_nh_list(old, new, extack);
1289         if (err)
1290                 return err;
1291
1292         err = fib6_check_nh_list(old, new, extack);
1293         if (err)
1294                 return err;
1295
1296         if (!new->is_group) {
1297                 struct nh_info *nhi = rtnl_dereference(new->nh_info);
1298
1299                 new_is_reject = nhi->reject_nh;
1300         }
1301
1302         list_for_each_entry(nhge, &old->grp_list, nh_list) {
1303                 /* if new nexthop is a blackhole, any groups using this
1304                  * nexthop cannot have more than 1 path
1305                  */
1306                 if (new_is_reject &&
1307                     nexthop_num_path(nhge->nh_parent) > 1) {
1308                         NL_SET_ERR_MSG(extack, "Blackhole nexthop can not be a member of a group with more than one path");
1309                         return -EINVAL;
1310                 }
1311
1312                 err = fib_check_nh_list(nhge->nh_parent, new, extack);
1313                 if (err)
1314                         return err;
1315
1316                 err = fib6_check_nh_list(nhge->nh_parent, new, extack);
1317                 if (err)
1318                         return err;
1319         }
1320
1321         if (old->is_group)
1322                 err = replace_nexthop_grp(net, old, new, extack);
1323         else
1324                 err = replace_nexthop_single(net, old, new, extack);
1325
1326         if (!err) {
1327                 nh_rt_cache_flush(net, old);
1328
1329                 __remove_nexthop(net, new, NULL);
1330                 nexthop_put(new);
1331         }
1332
1333         return err;
1334 }
1335
1336 /* called with rtnl_lock held */
1337 static int insert_nexthop(struct net *net, struct nexthop *new_nh,
1338                           struct nh_config *cfg, struct netlink_ext_ack *extack)
1339 {
1340         struct rb_node **pp, *parent = NULL, *next;
1341         struct rb_root *root = &net->nexthop.rb_root;
1342         bool replace = !!(cfg->nlflags & NLM_F_REPLACE);
1343         bool create = !!(cfg->nlflags & NLM_F_CREATE);
1344         u32 new_id = new_nh->id;
1345         int replace_notify = 0;
1346         int rc = -EEXIST;
1347
1348         pp = &root->rb_node;
1349         while (1) {
1350                 struct nexthop *nh;
1351
1352                 next = *pp;
1353                 if (!next)
1354                         break;
1355
1356                 parent = next;
1357
1358                 nh = rb_entry(parent, struct nexthop, rb_node);
1359                 if (new_id < nh->id) {
1360                         pp = &next->rb_left;
1361                 } else if (new_id > nh->id) {
1362                         pp = &next->rb_right;
1363                 } else if (replace) {
1364                         rc = replace_nexthop(net, nh, new_nh, extack);
1365                         if (!rc) {
1366                                 new_nh = nh; /* send notification with old nh */
1367                                 replace_notify = 1;
1368                         }
1369                         goto out;
1370                 } else {
1371                         /* id already exists and not a replace */
1372                         goto out;
1373                 }
1374         }
1375
1376         if (replace && !create) {
1377                 NL_SET_ERR_MSG(extack, "Replace specified without create and no entry exists");
1378                 rc = -ENOENT;
1379                 goto out;
1380         }
1381
1382         rb_link_node_rcu(&new_nh->rb_node, parent, pp);
1383         rb_insert_color(&new_nh->rb_node, root);
1384
1385         rc = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new_nh, extack);
1386         if (rc)
1387                 rb_erase(&new_nh->rb_node, &net->nexthop.rb_root);
1388
1389 out:
1390         if (!rc) {
1391                 nh_base_seq_inc(net);
1392                 nexthop_notify(RTM_NEWNEXTHOP, new_nh, &cfg->nlinfo);
1393                 if (replace_notify && net->ipv4.sysctl_nexthop_compat_mode)
1394                         nexthop_replace_notify(net, new_nh, &cfg->nlinfo);
1395         }
1396
1397         return rc;
1398 }
1399
1400 /* rtnl */
1401 /* remove all nexthops tied to a device being deleted */
1402 static void nexthop_flush_dev(struct net_device *dev)
1403 {
1404         unsigned int hash = nh_dev_hashfn(dev->ifindex);
1405         struct net *net = dev_net(dev);
1406         struct hlist_head *head = &net->nexthop.devhash[hash];
1407         struct hlist_node *n;
1408         struct nh_info *nhi;
1409
1410         hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
1411                 if (nhi->fib_nhc.nhc_dev != dev)
1412                         continue;
1413
1414                 remove_nexthop(net, nhi->nh_parent, NULL);
1415         }
1416 }
1417
1418 /* rtnl; called when net namespace is deleted */
1419 static void flush_all_nexthops(struct net *net)
1420 {
1421         struct rb_root *root = &net->nexthop.rb_root;
1422         struct rb_node *node;
1423         struct nexthop *nh;
1424
1425         while ((node = rb_first(root))) {
1426                 nh = rb_entry(node, struct nexthop, rb_node);
1427                 remove_nexthop(net, nh, NULL);
1428                 cond_resched();
1429         }
1430 }
1431
1432 static struct nexthop *nexthop_create_group(struct net *net,
1433                                             struct nh_config *cfg)
1434 {
1435         struct nlattr *grps_attr = cfg->nh_grp;
1436         struct nexthop_grp *entry = nla_data(grps_attr);
1437         u16 num_nh = nla_len(grps_attr) / sizeof(*entry);
1438         struct nh_group *nhg;
1439         struct nexthop *nh;
1440         int i;
1441
1442         if (WARN_ON(!num_nh))
1443                 return ERR_PTR(-EINVAL);
1444
1445         nh = nexthop_alloc();
1446         if (!nh)
1447                 return ERR_PTR(-ENOMEM);
1448
1449         nh->is_group = 1;
1450
1451         nhg = nexthop_grp_alloc(num_nh);
1452         if (!nhg) {
1453                 kfree(nh);
1454                 return ERR_PTR(-ENOMEM);
1455         }
1456
1457         /* spare group used for removals */
1458         nhg->spare = nexthop_grp_alloc(num_nh);
1459         if (!nhg->spare) {
1460                 kfree(nhg);
1461                 kfree(nh);
1462                 return ERR_PTR(-ENOMEM);
1463         }
1464         nhg->spare->spare = nhg;
1465
1466         for (i = 0; i < nhg->num_nh; ++i) {
1467                 struct nexthop *nhe;
1468                 struct nh_info *nhi;
1469
1470                 nhe = nexthop_find_by_id(net, entry[i].id);
1471                 if (!nexthop_get(nhe))
1472                         goto out_no_nh;
1473
1474                 nhi = rtnl_dereference(nhe->nh_info);
1475                 if (nhi->family == AF_INET)
1476                         nhg->has_v4 = true;
1477
1478                 nhg->nh_entries[i].nh = nhe;
1479                 nhg->nh_entries[i].weight = entry[i].weight + 1;
1480                 list_add(&nhg->nh_entries[i].nh_list, &nhe->grp_list);
1481                 nhg->nh_entries[i].nh_parent = nh;
1482         }
1483
1484         if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_MPATH)
1485                 nhg->mpath = 1;
1486
1487         WARN_ON_ONCE(nhg->mpath != 1);
1488
1489         if (nhg->mpath)
1490                 nh_group_rebalance(nhg);
1491
1492         if (cfg->nh_fdb)
1493                 nhg->fdb_nh = 1;
1494
1495         rcu_assign_pointer(nh->nh_grp, nhg);
1496
1497         return nh;
1498
1499 out_no_nh:
1500         for (i--; i >= 0; --i) {
1501                 list_del(&nhg->nh_entries[i].nh_list);
1502                 nexthop_put(nhg->nh_entries[i].nh);
1503         }
1504
1505         kfree(nhg->spare);
1506         kfree(nhg);
1507         kfree(nh);
1508
1509         return ERR_PTR(-ENOENT);
1510 }
1511
1512 static int nh_create_ipv4(struct net *net, struct nexthop *nh,
1513                           struct nh_info *nhi, struct nh_config *cfg,
1514                           struct netlink_ext_ack *extack)
1515 {
1516         struct fib_nh *fib_nh = &nhi->fib_nh;
1517         struct fib_config fib_cfg = {
1518                 .fc_oif   = cfg->nh_ifindex,
1519                 .fc_gw4   = cfg->gw.ipv4,
1520                 .fc_gw_family = cfg->gw.ipv4 ? AF_INET : 0,
1521                 .fc_flags = cfg->nh_flags,
1522                 .fc_encap = cfg->nh_encap,
1523                 .fc_encap_type = cfg->nh_encap_type,
1524         };
1525         u32 tb_id = (cfg->dev ? l3mdev_fib_table(cfg->dev) : RT_TABLE_MAIN);
1526         int err;
1527
1528         err = fib_nh_init(net, fib_nh, &fib_cfg, 1, extack);
1529         if (err) {
1530                 fib_nh_release(net, fib_nh);
1531                 goto out;
1532         }
1533
1534         if (nhi->fdb_nh)
1535                 goto out;
1536
1537         /* sets nh_dev if successful */
1538         err = fib_check_nh(net, fib_nh, tb_id, 0, extack);
1539         if (!err) {
1540                 nh->nh_flags = fib_nh->fib_nh_flags;
1541                 fib_info_update_nhc_saddr(net, &fib_nh->nh_common,
1542                                           fib_nh->fib_nh_scope);
1543         } else {
1544                 fib_nh_release(net, fib_nh);
1545         }
1546 out:
1547         return err;
1548 }
1549
1550 static int nh_create_ipv6(struct net *net,  struct nexthop *nh,
1551                           struct nh_info *nhi, struct nh_config *cfg,
1552                           struct netlink_ext_ack *extack)
1553 {
1554         struct fib6_nh *fib6_nh = &nhi->fib6_nh;
1555         struct fib6_config fib6_cfg = {
1556                 .fc_table = l3mdev_fib_table(cfg->dev),
1557                 .fc_ifindex = cfg->nh_ifindex,
1558                 .fc_gateway = cfg->gw.ipv6,
1559                 .fc_flags = cfg->nh_flags,
1560                 .fc_encap = cfg->nh_encap,
1561                 .fc_encap_type = cfg->nh_encap_type,
1562                 .fc_is_fdb = cfg->nh_fdb,
1563         };
1564         int err;
1565
1566         if (!ipv6_addr_any(&cfg->gw.ipv6))
1567                 fib6_cfg.fc_flags |= RTF_GATEWAY;
1568
1569         /* sets nh_dev if successful */
1570         err = ipv6_stub->fib6_nh_init(net, fib6_nh, &fib6_cfg, GFP_KERNEL,
1571                                       extack);
1572         if (err)
1573                 ipv6_stub->fib6_nh_release(fib6_nh);
1574         else
1575                 nh->nh_flags = fib6_nh->fib_nh_flags;
1576
1577         return err;
1578 }
1579
1580 static struct nexthop *nexthop_create(struct net *net, struct nh_config *cfg,
1581                                       struct netlink_ext_ack *extack)
1582 {
1583         struct nh_info *nhi;
1584         struct nexthop *nh;
1585         int err = 0;
1586
1587         nh = nexthop_alloc();
1588         if (!nh)
1589                 return ERR_PTR(-ENOMEM);
1590
1591         nhi = kzalloc(sizeof(*nhi), GFP_KERNEL);
1592         if (!nhi) {
1593                 kfree(nh);
1594                 return ERR_PTR(-ENOMEM);
1595         }
1596
1597         nh->nh_flags = cfg->nh_flags;
1598         nh->net = net;
1599
1600         nhi->nh_parent = nh;
1601         nhi->family = cfg->nh_family;
1602         nhi->fib_nhc.nhc_scope = RT_SCOPE_LINK;
1603
1604         if (cfg->nh_fdb)
1605                 nhi->fdb_nh = 1;
1606
1607         if (cfg->nh_blackhole) {
1608                 nhi->reject_nh = 1;
1609                 cfg->nh_ifindex = net->loopback_dev->ifindex;
1610         }
1611
1612         switch (cfg->nh_family) {
1613         case AF_INET:
1614                 err = nh_create_ipv4(net, nh, nhi, cfg, extack);
1615                 break;
1616         case AF_INET6:
1617                 err = nh_create_ipv6(net, nh, nhi, cfg, extack);
1618                 break;
1619         }
1620
1621         if (err) {
1622                 kfree(nhi);
1623                 kfree(nh);
1624                 return ERR_PTR(err);
1625         }
1626
1627         /* add the entry to the device based hash */
1628         if (!nhi->fdb_nh)
1629                 nexthop_devhash_add(net, nhi);
1630
1631         rcu_assign_pointer(nh->nh_info, nhi);
1632
1633         return nh;
1634 }
1635
1636 /* called with rtnl lock held */
1637 static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg,
1638                                    struct netlink_ext_ack *extack)
1639 {
1640         struct nexthop *nh;
1641         int err;
1642
1643         if (cfg->nlflags & NLM_F_REPLACE && !cfg->nh_id) {
1644                 NL_SET_ERR_MSG(extack, "Replace requires nexthop id");
1645                 return ERR_PTR(-EINVAL);
1646         }
1647
1648         if (!cfg->nh_id) {
1649                 cfg->nh_id = nh_find_unused_id(net);
1650                 if (!cfg->nh_id) {
1651                         NL_SET_ERR_MSG(extack, "No unused id");
1652                         return ERR_PTR(-EINVAL);
1653                 }
1654         }
1655
1656         if (cfg->nh_grp)
1657                 nh = nexthop_create_group(net, cfg);
1658         else
1659                 nh = nexthop_create(net, cfg, extack);
1660
1661         if (IS_ERR(nh))
1662                 return nh;
1663
1664         refcount_set(&nh->refcnt, 1);
1665         nh->id = cfg->nh_id;
1666         nh->protocol = cfg->nh_protocol;
1667         nh->net = net;
1668
1669         err = insert_nexthop(net, nh, cfg, extack);
1670         if (err) {
1671                 __remove_nexthop(net, nh, NULL);
1672                 nexthop_put(nh);
1673                 nh = ERR_PTR(err);
1674         }
1675
1676         return nh;
1677 }
1678
1679 static int rtm_to_nh_config(struct net *net, struct sk_buff *skb,
1680                             struct nlmsghdr *nlh, struct nh_config *cfg,
1681                             struct netlink_ext_ack *extack)
1682 {
1683         struct nhmsg *nhm = nlmsg_data(nlh);
1684         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_new)];
1685         int err;
1686
1687         err = nlmsg_parse(nlh, sizeof(*nhm), tb,
1688                           ARRAY_SIZE(rtm_nh_policy_new) - 1,
1689                           rtm_nh_policy_new, extack);
1690         if (err < 0)
1691                 return err;
1692
1693         err = -EINVAL;
1694         if (nhm->resvd || nhm->nh_scope) {
1695                 NL_SET_ERR_MSG(extack, "Invalid values in ancillary header");
1696                 goto out;
1697         }
1698         if (nhm->nh_flags & ~NEXTHOP_VALID_USER_FLAGS) {
1699                 NL_SET_ERR_MSG(extack, "Invalid nexthop flags in ancillary header");
1700                 goto out;
1701         }
1702
1703         switch (nhm->nh_family) {
1704         case AF_INET:
1705         case AF_INET6:
1706                 break;
1707         case AF_UNSPEC:
1708                 if (tb[NHA_GROUP])
1709                         break;
1710                 fallthrough;
1711         default:
1712                 NL_SET_ERR_MSG(extack, "Invalid address family");
1713                 goto out;
1714         }
1715
1716         memset(cfg, 0, sizeof(*cfg));
1717         cfg->nlflags = nlh->nlmsg_flags;
1718         cfg->nlinfo.portid = NETLINK_CB(skb).portid;
1719         cfg->nlinfo.nlh = nlh;
1720         cfg->nlinfo.nl_net = net;
1721
1722         cfg->nh_family = nhm->nh_family;
1723         cfg->nh_protocol = nhm->nh_protocol;
1724         cfg->nh_flags = nhm->nh_flags;
1725
1726         if (tb[NHA_ID])
1727                 cfg->nh_id = nla_get_u32(tb[NHA_ID]);
1728
1729         if (tb[NHA_FDB]) {
1730                 if (tb[NHA_OIF] || tb[NHA_BLACKHOLE] ||
1731                     tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE]) {
1732                         NL_SET_ERR_MSG(extack, "Fdb attribute can not be used with encap, oif or blackhole");
1733                         goto out;
1734                 }
1735                 if (nhm->nh_flags) {
1736                         NL_SET_ERR_MSG(extack, "Unsupported nexthop flags in ancillary header");
1737                         goto out;
1738                 }
1739                 cfg->nh_fdb = nla_get_flag(tb[NHA_FDB]);
1740         }
1741
1742         if (tb[NHA_GROUP]) {
1743                 if (nhm->nh_family != AF_UNSPEC) {
1744                         NL_SET_ERR_MSG(extack, "Invalid family for group");
1745                         goto out;
1746                 }
1747                 cfg->nh_grp = tb[NHA_GROUP];
1748
1749                 cfg->nh_grp_type = NEXTHOP_GRP_TYPE_MPATH;
1750                 if (tb[NHA_GROUP_TYPE])
1751                         cfg->nh_grp_type = nla_get_u16(tb[NHA_GROUP_TYPE]);
1752
1753                 if (cfg->nh_grp_type > NEXTHOP_GRP_TYPE_MAX) {
1754                         NL_SET_ERR_MSG(extack, "Invalid group type");
1755                         goto out;
1756                 }
1757                 err = nh_check_attr_group(net, tb, ARRAY_SIZE(tb), extack);
1758
1759                 /* no other attributes should be set */
1760                 goto out;
1761         }
1762
1763         if (tb[NHA_BLACKHOLE]) {
1764                 if (tb[NHA_GATEWAY] || tb[NHA_OIF] ||
1765                     tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE] || tb[NHA_FDB]) {
1766                         NL_SET_ERR_MSG(extack, "Blackhole attribute can not be used with gateway, oif, encap or fdb");
1767                         goto out;
1768                 }
1769
1770                 cfg->nh_blackhole = 1;
1771                 err = 0;
1772                 goto out;
1773         }
1774
1775         if (!cfg->nh_fdb && !tb[NHA_OIF]) {
1776                 NL_SET_ERR_MSG(extack, "Device attribute required for non-blackhole and non-fdb nexthops");
1777                 goto out;
1778         }
1779
1780         if (!cfg->nh_fdb && tb[NHA_OIF]) {
1781                 cfg->nh_ifindex = nla_get_u32(tb[NHA_OIF]);
1782                 if (cfg->nh_ifindex)
1783                         cfg->dev = __dev_get_by_index(net, cfg->nh_ifindex);
1784
1785                 if (!cfg->dev) {
1786                         NL_SET_ERR_MSG(extack, "Invalid device index");
1787                         goto out;
1788                 } else if (!(cfg->dev->flags & IFF_UP)) {
1789                         NL_SET_ERR_MSG(extack, "Nexthop device is not up");
1790                         err = -ENETDOWN;
1791                         goto out;
1792                 } else if (!netif_carrier_ok(cfg->dev)) {
1793                         NL_SET_ERR_MSG(extack, "Carrier for nexthop device is down");
1794                         err = -ENETDOWN;
1795                         goto out;
1796                 }
1797         }
1798
1799         err = -EINVAL;
1800         if (tb[NHA_GATEWAY]) {
1801                 struct nlattr *gwa = tb[NHA_GATEWAY];
1802
1803                 switch (cfg->nh_family) {
1804                 case AF_INET:
1805                         if (nla_len(gwa) != sizeof(u32)) {
1806                                 NL_SET_ERR_MSG(extack, "Invalid gateway");
1807                                 goto out;
1808                         }
1809                         cfg->gw.ipv4 = nla_get_be32(gwa);
1810                         break;
1811                 case AF_INET6:
1812                         if (nla_len(gwa) != sizeof(struct in6_addr)) {
1813                                 NL_SET_ERR_MSG(extack, "Invalid gateway");
1814                                 goto out;
1815                         }
1816                         cfg->gw.ipv6 = nla_get_in6_addr(gwa);
1817                         break;
1818                 default:
1819                         NL_SET_ERR_MSG(extack,
1820                                        "Unknown address family for gateway");
1821                         goto out;
1822                 }
1823         } else {
1824                 /* device only nexthop (no gateway) */
1825                 if (cfg->nh_flags & RTNH_F_ONLINK) {
1826                         NL_SET_ERR_MSG(extack,
1827                                        "ONLINK flag can not be set for nexthop without a gateway");
1828                         goto out;
1829                 }
1830         }
1831
1832         if (tb[NHA_ENCAP]) {
1833                 cfg->nh_encap = tb[NHA_ENCAP];
1834
1835                 if (!tb[NHA_ENCAP_TYPE]) {
1836                         NL_SET_ERR_MSG(extack, "LWT encapsulation type is missing");
1837                         goto out;
1838                 }
1839
1840                 cfg->nh_encap_type = nla_get_u16(tb[NHA_ENCAP_TYPE]);
1841                 err = lwtunnel_valid_encap_type(cfg->nh_encap_type, extack);
1842                 if (err < 0)
1843                         goto out;
1844
1845         } else if (tb[NHA_ENCAP_TYPE]) {
1846                 NL_SET_ERR_MSG(extack, "LWT encapsulation attribute is missing");
1847                 goto out;
1848         }
1849
1850
1851         err = 0;
1852 out:
1853         return err;
1854 }
1855
1856 /* rtnl */
1857 static int rtm_new_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
1858                            struct netlink_ext_ack *extack)
1859 {
1860         struct net *net = sock_net(skb->sk);
1861         struct nh_config cfg;
1862         struct nexthop *nh;
1863         int err;
1864
1865         err = rtm_to_nh_config(net, skb, nlh, &cfg, extack);
1866         if (!err) {
1867                 nh = nexthop_add(net, &cfg, extack);
1868                 if (IS_ERR(nh))
1869                         err = PTR_ERR(nh);
1870         }
1871
1872         return err;
1873 }
1874
1875 static int __nh_valid_get_del_req(const struct nlmsghdr *nlh,
1876                                   struct nlattr **tb, u32 *id,
1877                                   struct netlink_ext_ack *extack)
1878 {
1879         struct nhmsg *nhm = nlmsg_data(nlh);
1880
1881         if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
1882                 NL_SET_ERR_MSG(extack, "Invalid values in header");
1883                 return -EINVAL;
1884         }
1885
1886         if (!tb[NHA_ID]) {
1887                 NL_SET_ERR_MSG(extack, "Nexthop id is missing");
1888                 return -EINVAL;
1889         }
1890
1891         *id = nla_get_u32(tb[NHA_ID]);
1892         if (!(*id)) {
1893                 NL_SET_ERR_MSG(extack, "Invalid nexthop id");
1894                 return -EINVAL;
1895         }
1896
1897         return 0;
1898 }
1899
1900 static int nh_valid_get_del_req(const struct nlmsghdr *nlh, u32 *id,
1901                                 struct netlink_ext_ack *extack)
1902 {
1903         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_get)];
1904         int err;
1905
1906         err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
1907                           ARRAY_SIZE(rtm_nh_policy_get) - 1,
1908                           rtm_nh_policy_get, extack);
1909         if (err < 0)
1910                 return err;
1911
1912         return __nh_valid_get_del_req(nlh, tb, id, extack);
1913 }
1914
1915 /* rtnl */
1916 static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
1917                            struct netlink_ext_ack *extack)
1918 {
1919         struct net *net = sock_net(skb->sk);
1920         struct nl_info nlinfo = {
1921                 .nlh = nlh,
1922                 .nl_net = net,
1923                 .portid = NETLINK_CB(skb).portid,
1924         };
1925         struct nexthop *nh;
1926         int err;
1927         u32 id;
1928
1929         err = nh_valid_get_del_req(nlh, &id, extack);
1930         if (err)
1931                 return err;
1932
1933         nh = nexthop_find_by_id(net, id);
1934         if (!nh)
1935                 return -ENOENT;
1936
1937         remove_nexthop(net, nh, &nlinfo);
1938
1939         return 0;
1940 }
1941
1942 /* rtnl */
1943 static int rtm_get_nexthop(struct sk_buff *in_skb, struct nlmsghdr *nlh,
1944                            struct netlink_ext_ack *extack)
1945 {
1946         struct net *net = sock_net(in_skb->sk);
1947         struct sk_buff *skb = NULL;
1948         struct nexthop *nh;
1949         int err;
1950         u32 id;
1951
1952         err = nh_valid_get_del_req(nlh, &id, extack);
1953         if (err)
1954                 return err;
1955
1956         err = -ENOBUFS;
1957         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1958         if (!skb)
1959                 goto out;
1960
1961         err = -ENOENT;
1962         nh = nexthop_find_by_id(net, id);
1963         if (!nh)
1964                 goto errout_free;
1965
1966         err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP, NETLINK_CB(in_skb).portid,
1967                            nlh->nlmsg_seq, 0);
1968         if (err < 0) {
1969                 WARN_ON(err == -EMSGSIZE);
1970                 goto errout_free;
1971         }
1972
1973         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
1974 out:
1975         return err;
1976 errout_free:
1977         kfree_skb(skb);
1978         goto out;
1979 }
1980
1981 struct nh_dump_filter {
1982         int dev_idx;
1983         int master_idx;
1984         bool group_filter;
1985         bool fdb_filter;
1986 };
1987
1988 static bool nh_dump_filtered(struct nexthop *nh,
1989                              struct nh_dump_filter *filter, u8 family)
1990 {
1991         const struct net_device *dev;
1992         const struct nh_info *nhi;
1993
1994         if (filter->group_filter && !nh->is_group)
1995                 return true;
1996
1997         if (!filter->dev_idx && !filter->master_idx && !family)
1998                 return false;
1999
2000         if (nh->is_group)
2001                 return true;
2002
2003         nhi = rtnl_dereference(nh->nh_info);
2004         if (family && nhi->family != family)
2005                 return true;
2006
2007         dev = nhi->fib_nhc.nhc_dev;
2008         if (filter->dev_idx && (!dev || dev->ifindex != filter->dev_idx))
2009                 return true;
2010
2011         if (filter->master_idx) {
2012                 struct net_device *master;
2013
2014                 if (!dev)
2015                         return true;
2016
2017                 master = netdev_master_upper_dev_get((struct net_device *)dev);
2018                 if (!master || master->ifindex != filter->master_idx)
2019                         return true;
2020         }
2021
2022         return false;
2023 }
2024
2025 static int __nh_valid_dump_req(const struct nlmsghdr *nlh, struct nlattr **tb,
2026                                struct nh_dump_filter *filter,
2027                                struct netlink_ext_ack *extack)
2028 {
2029         struct nhmsg *nhm;
2030         u32 idx;
2031
2032         if (tb[NHA_OIF]) {
2033                 idx = nla_get_u32(tb[NHA_OIF]);
2034                 if (idx > INT_MAX) {
2035                         NL_SET_ERR_MSG(extack, "Invalid device index");
2036                         return -EINVAL;
2037                 }
2038                 filter->dev_idx = idx;
2039         }
2040         if (tb[NHA_MASTER]) {
2041                 idx = nla_get_u32(tb[NHA_MASTER]);
2042                 if (idx > INT_MAX) {
2043                         NL_SET_ERR_MSG(extack, "Invalid master device index");
2044                         return -EINVAL;
2045                 }
2046                 filter->master_idx = idx;
2047         }
2048         filter->group_filter = nla_get_flag(tb[NHA_GROUPS]);
2049         filter->fdb_filter = nla_get_flag(tb[NHA_FDB]);
2050
2051         nhm = nlmsg_data(nlh);
2052         if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
2053                 NL_SET_ERR_MSG(extack, "Invalid values in header for nexthop dump request");
2054                 return -EINVAL;
2055         }
2056
2057         return 0;
2058 }
2059
2060 static int nh_valid_dump_req(const struct nlmsghdr *nlh,
2061                              struct nh_dump_filter *filter,
2062                              struct netlink_callback *cb)
2063 {
2064         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_dump)];
2065         int err;
2066
2067         err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
2068                           ARRAY_SIZE(rtm_nh_policy_dump) - 1,
2069                           rtm_nh_policy_dump, cb->extack);
2070         if (err < 0)
2071                 return err;
2072
2073         return __nh_valid_dump_req(nlh, tb, filter, cb->extack);
2074 }
2075
2076 struct rtm_dump_nh_ctx {
2077         u32 idx;
2078 };
2079
2080 static struct rtm_dump_nh_ctx *
2081 rtm_dump_nh_ctx(struct netlink_callback *cb)
2082 {
2083         struct rtm_dump_nh_ctx *ctx = (void *)cb->ctx;
2084
2085         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
2086         return ctx;
2087 }
2088
2089 static int rtm_dump_walk_nexthops(struct sk_buff *skb,
2090                                   struct netlink_callback *cb,
2091                                   struct rb_root *root,
2092                                   struct rtm_dump_nh_ctx *ctx,
2093                                   int (*nh_cb)(struct sk_buff *skb,
2094                                                struct netlink_callback *cb,
2095                                                struct nexthop *nh, void *data),
2096                                   void *data)
2097 {
2098         struct rb_node *node;
2099         int idx = 0, s_idx;
2100         int err;
2101
2102         s_idx = ctx->idx;
2103         for (node = rb_first(root); node; node = rb_next(node)) {
2104                 struct nexthop *nh;
2105
2106                 if (idx < s_idx)
2107                         goto cont;
2108
2109                 nh = rb_entry(node, struct nexthop, rb_node);
2110                 ctx->idx = idx;
2111                 err = nh_cb(skb, cb, nh, data);
2112                 if (err)
2113                         return err;
2114 cont:
2115                 idx++;
2116         }
2117
2118         ctx->idx = idx;
2119         return 0;
2120 }
2121
2122 static int rtm_dump_nexthop_cb(struct sk_buff *skb, struct netlink_callback *cb,
2123                                struct nexthop *nh, void *data)
2124 {
2125         struct nhmsg *nhm = nlmsg_data(cb->nlh);
2126         struct nh_dump_filter *filter = data;
2127
2128         if (nh_dump_filtered(nh, filter, nhm->nh_family))
2129                 return 0;
2130
2131         return nh_fill_node(skb, nh, RTM_NEWNEXTHOP,
2132                             NETLINK_CB(cb->skb).portid,
2133                             cb->nlh->nlmsg_seq, NLM_F_MULTI);
2134 }
2135
2136 /* rtnl */
2137 static int rtm_dump_nexthop(struct sk_buff *skb, struct netlink_callback *cb)
2138 {
2139         struct rtm_dump_nh_ctx *ctx = rtm_dump_nh_ctx(cb);
2140         struct net *net = sock_net(skb->sk);
2141         struct rb_root *root = &net->nexthop.rb_root;
2142         struct nh_dump_filter filter = {};
2143         int err;
2144
2145         err = nh_valid_dump_req(cb->nlh, &filter, cb);
2146         if (err < 0)
2147                 return err;
2148
2149         err = rtm_dump_walk_nexthops(skb, cb, root, ctx,
2150                                      &rtm_dump_nexthop_cb, &filter);
2151         if (err < 0) {
2152                 if (likely(skb->len))
2153                         goto out;
2154                 goto out_err;
2155         }
2156
2157 out:
2158         err = skb->len;
2159 out_err:
2160         cb->seq = net->nexthop.seq;
2161         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
2162         return err;
2163 }
2164
2165 static void nexthop_sync_mtu(struct net_device *dev, u32 orig_mtu)
2166 {
2167         unsigned int hash = nh_dev_hashfn(dev->ifindex);
2168         struct net *net = dev_net(dev);
2169         struct hlist_head *head = &net->nexthop.devhash[hash];
2170         struct hlist_node *n;
2171         struct nh_info *nhi;
2172
2173         hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
2174                 if (nhi->fib_nhc.nhc_dev == dev) {
2175                         if (nhi->family == AF_INET)
2176                                 fib_nhc_update_mtu(&nhi->fib_nhc, dev->mtu,
2177                                                    orig_mtu);
2178                 }
2179         }
2180 }
2181
2182 /* rtnl */
2183 static int nh_netdev_event(struct notifier_block *this,
2184                            unsigned long event, void *ptr)
2185 {
2186         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
2187         struct netdev_notifier_info_ext *info_ext;
2188
2189         switch (event) {
2190         case NETDEV_DOWN:
2191         case NETDEV_UNREGISTER:
2192                 nexthop_flush_dev(dev);
2193                 break;
2194         case NETDEV_CHANGE:
2195                 if (!(dev_get_flags(dev) & (IFF_RUNNING | IFF_LOWER_UP)))
2196                         nexthop_flush_dev(dev);
2197                 break;
2198         case NETDEV_CHANGEMTU:
2199                 info_ext = ptr;
2200                 nexthop_sync_mtu(dev, info_ext->ext.mtu);
2201                 rt_cache_flush(dev_net(dev));
2202                 break;
2203         }
2204         return NOTIFY_DONE;
2205 }
2206
2207 static struct notifier_block nh_netdev_notifier = {
2208         .notifier_call = nh_netdev_event,
2209 };
2210
2211 static int nexthops_dump(struct net *net, struct notifier_block *nb,
2212                          struct netlink_ext_ack *extack)
2213 {
2214         struct rb_root *root = &net->nexthop.rb_root;
2215         struct rb_node *node;
2216         int err = 0;
2217
2218         for (node = rb_first(root); node; node = rb_next(node)) {
2219                 struct nexthop *nh;
2220
2221                 nh = rb_entry(node, struct nexthop, rb_node);
2222                 err = call_nexthop_notifier(nb, net, NEXTHOP_EVENT_REPLACE, nh,
2223                                             extack);
2224                 if (err)
2225                         break;
2226         }
2227
2228         return err;
2229 }
2230
2231 int register_nexthop_notifier(struct net *net, struct notifier_block *nb,
2232                               struct netlink_ext_ack *extack)
2233 {
2234         int err;
2235
2236         rtnl_lock();
2237         err = nexthops_dump(net, nb, extack);
2238         if (err)
2239                 goto unlock;
2240         err = blocking_notifier_chain_register(&net->nexthop.notifier_chain,
2241                                                nb);
2242 unlock:
2243         rtnl_unlock();
2244         return err;
2245 }
2246 EXPORT_SYMBOL(register_nexthop_notifier);
2247
2248 int unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
2249 {
2250         return blocking_notifier_chain_unregister(&net->nexthop.notifier_chain,
2251                                                   nb);
2252 }
2253 EXPORT_SYMBOL(unregister_nexthop_notifier);
2254
2255 void nexthop_set_hw_flags(struct net *net, u32 id, bool offload, bool trap)
2256 {
2257         struct nexthop *nexthop;
2258
2259         rcu_read_lock();
2260
2261         nexthop = nexthop_find_by_id(net, id);
2262         if (!nexthop)
2263                 goto out;
2264
2265         nexthop->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
2266         if (offload)
2267                 nexthop->nh_flags |= RTNH_F_OFFLOAD;
2268         if (trap)
2269                 nexthop->nh_flags |= RTNH_F_TRAP;
2270
2271 out:
2272         rcu_read_unlock();
2273 }
2274 EXPORT_SYMBOL(nexthop_set_hw_flags);
2275
2276 static void __net_exit nexthop_net_exit(struct net *net)
2277 {
2278         rtnl_lock();
2279         flush_all_nexthops(net);
2280         rtnl_unlock();
2281         kfree(net->nexthop.devhash);
2282 }
2283
2284 static int __net_init nexthop_net_init(struct net *net)
2285 {
2286         size_t sz = sizeof(struct hlist_head) * NH_DEV_HASHSIZE;
2287
2288         net->nexthop.rb_root = RB_ROOT;
2289         net->nexthop.devhash = kzalloc(sz, GFP_KERNEL);
2290         if (!net->nexthop.devhash)
2291                 return -ENOMEM;
2292         BLOCKING_INIT_NOTIFIER_HEAD(&net->nexthop.notifier_chain);
2293
2294         return 0;
2295 }
2296
2297 static struct pernet_operations nexthop_net_ops = {
2298         .init = nexthop_net_init,
2299         .exit = nexthop_net_exit,
2300 };
2301
2302 static int __init nexthop_init(void)
2303 {
2304         register_pernet_subsys(&nexthop_net_ops);
2305
2306         register_netdevice_notifier(&nh_netdev_notifier);
2307
2308         rtnl_register(PF_UNSPEC, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
2309         rtnl_register(PF_UNSPEC, RTM_DELNEXTHOP, rtm_del_nexthop, NULL, 0);
2310         rtnl_register(PF_UNSPEC, RTM_GETNEXTHOP, rtm_get_nexthop,
2311                       rtm_dump_nexthop, 0);
2312
2313         rtnl_register(PF_INET, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
2314         rtnl_register(PF_INET, RTM_GETNEXTHOP, NULL, rtm_dump_nexthop, 0);
2315
2316         rtnl_register(PF_INET6, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
2317         rtnl_register(PF_INET6, RTM_GETNEXTHOP, NULL, rtm_dump_nexthop, 0);
2318
2319         return 0;
2320 }
2321 subsys_initcall(nexthop_init);