drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <linux/inetdevice.h>
  44 #include <linux/netdevice.h>
  45 #include <linux/if_bridge.h>
  46 #include <linux/socket.h>
  47 #include <linux/route.h>
  48 #include <net/netevent.h>
  49 #include <net/neighbour.h>
  50 #include <net/arp.h>
  51 #include <net/ip_fib.h>
  52 #include <net/ip6_fib.h>
  53 #include <net/fib_rules.h>
  54 #include <net/l3mdev.h>
  55 #include <net/addrconf.h>
  56 #include <net/ndisc.h>
  57 #include <net/ipv6.h>
  58 #include <net/fib_notifier.h>
  59
  60 #include "spectrum.h"
  61 #include "core.h"
  62 #include "reg.h"
  63 #include "spectrum_cnt.h"
  64 #include "spectrum_dpipe.h"
  65 #include "spectrum_router.h"
  66
  67 struct mlxsw_sp_vr;
  68 struct mlxsw_sp_lpm_tree;
  69 struct mlxsw_sp_rif_ops;
  70
  71 struct mlxsw_sp_router {
  72         struct mlxsw_sp *mlxsw_sp;
  73         struct mlxsw_sp_rif **rifs;
  74         struct mlxsw_sp_vr *vrs;
  75         struct rhashtable neigh_ht;
  76         struct rhashtable nexthop_group_ht;
  77         struct rhashtable nexthop_ht;
  78         struct {
  79                 struct mlxsw_sp_lpm_tree *trees;
  80                 unsigned int tree_count;
  81         } lpm;
  82         struct {
  83                 struct delayed_work dw;
  84                 unsigned long interval; /* ms */
  85         } neighs_update;
  86         struct delayed_work nexthop_probe_dw;
  87 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  88         struct list_head nexthop_neighs_list;
  89         bool aborted;
  90         struct notifier_block fib_nb;
  91         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  92 };
  93
  94 struct mlxsw_sp_rif {
  95         struct list_head nexthop_list;
  96         struct list_head neigh_list;
  97         struct net_device *dev;
  98         struct mlxsw_sp_fid *fid;
  99         unsigned char addr[ETH_ALEN];
 100         int mtu;
 101         u16 rif_index;
 102         u16 vr_id;
 103         const struct mlxsw_sp_rif_ops *ops;
 104         struct mlxsw_sp *mlxsw_sp;
 105
 106         unsigned int counter_ingress;
 107         bool counter_ingress_valid;
 108         unsigned int counter_egress;
 109         bool counter_egress_valid;
 110 };
 111
 112 struct mlxsw_sp_rif_params {
 113         struct net_device *dev;
 114         union {
 115                 u16 system_port;
 116                 u16 lag_id;
 117         };
 118         u16 vid;
 119         bool lag;
 120 };
 121
 122 struct mlxsw_sp_rif_subport {
 123         struct mlxsw_sp_rif common;
 124         union {
 125                 u16 system_port;
 126                 u16 lag_id;
 127         };
 128         u16 vid;
 129         bool lag;
 130 };
 131
 132 struct mlxsw_sp_rif_ops {
 133         enum mlxsw_sp_rif_type type;
 134         size_t rif_size;
 135
 136         void (*setup)(struct mlxsw_sp_rif *rif,
 137                       const struct mlxsw_sp_rif_params *params);
 138         int (*configure)(struct mlxsw_sp_rif *rif);
 139         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 140         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 141 };
 142
 143 static unsigned int *
 144 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 145                            enum mlxsw_sp_rif_counter_dir dir)
 146 {
 147         switch (dir) {
 148         case MLXSW_SP_RIF_COUNTER_EGRESS:
 149                 return &rif->counter_egress;
 150         case MLXSW_SP_RIF_COUNTER_INGRESS:
 151                 return &rif->counter_ingress;
 152         }
 153         return NULL;
 154 }
 155
 156 static bool
 157 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 158                                enum mlxsw_sp_rif_counter_dir dir)
 159 {
 160         switch (dir) {
 161         case MLXSW_SP_RIF_COUNTER_EGRESS:
 162                 return rif->counter_egress_valid;
 163         case MLXSW_SP_RIF_COUNTER_INGRESS:
 164                 return rif->counter_ingress_valid;
 165         }
 166         return false;
 167 }
 168
 169 static void
 170 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 171                                enum mlxsw_sp_rif_counter_dir dir,
 172                                bool valid)
 173 {
 174         switch (dir) {
 175         case MLXSW_SP_RIF_COUNTER_EGRESS:
 176                 rif->counter_egress_valid = valid;
 177                 break;
 178         case MLXSW_SP_RIF_COUNTER_INGRESS:
 179                 rif->counter_ingress_valid = valid;
 180                 break;
 181         }
 182 }
 183
 184 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 185                                      unsigned int counter_index, bool enable,
 186                                      enum mlxsw_sp_rif_counter_dir dir)
 187 {
 188         char ritr_pl[MLXSW_REG_RITR_LEN];
 189         bool is_egress = false;
 190         int err;
 191
 192         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 193                 is_egress = true;
 194         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 195         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 196         if (err)
 197                 return err;
 198
 199         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 200                                     is_egress);
 201         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 202 }
 203
 204 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 205                                    struct mlxsw_sp_rif *rif,
 206                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 207 {
 208         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 209         unsigned int *p_counter_index;
 210         bool valid;
 211         int err;
 212
 213         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 214         if (!valid)
 215                 return -EINVAL;
 216
 217         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 218         if (!p_counter_index)
 219                 return -EINVAL;
 220         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 221                              MLXSW_REG_RICNT_OPCODE_NOP);
 222         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 223         if (err)
 224                 return err;
 225         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 226         return 0;
 227 }
 228
 229 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 230                                       unsigned int counter_index)
 231 {
 232         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 233
 234         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 235                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 236         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 237 }
 238
 239 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 240                                struct mlxsw_sp_rif *rif,
 241                                enum mlxsw_sp_rif_counter_dir dir)
 242 {
 243         unsigned int *p_counter_index;
 244         int err;
 245
 246         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 247         if (!p_counter_index)
 248                 return -EINVAL;
 249         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 250                                      p_counter_index);
 251         if (err)
 252                 return err;
 253
 254         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 255         if (err)
 256                 goto err_counter_clear;
 257
 258         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 259                                         *p_counter_index, true, dir);
 260         if (err)
 261                 goto err_counter_edit;
 262         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 263         return 0;
 264
 265 err_counter_edit:
 266 err_counter_clear:
 267         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 268                               *p_counter_index);
 269         return err;
 270 }
 271
 272 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 273                                struct mlxsw_sp_rif *rif,
 274                                enum mlxsw_sp_rif_counter_dir dir)
 275 {
 276         unsigned int *p_counter_index;
 277
 278         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 279                 return;
 280
 281         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 282         if (WARN_ON(!p_counter_index))
 283                 return;
 284         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 285                                   *p_counter_index, false, dir);
 286         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 287                               *p_counter_index);
 288         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 289 }
 290
 291 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 292 {
 293         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 294         struct devlink *devlink;
 295
 296         devlink = priv_to_devlink(mlxsw_sp->core);
 297         if (!devlink_dpipe_table_counter_enabled(devlink,
 298                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 299                 return;
 300         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 301 }
 302
 303 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 304 {
 305         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 306
 307         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 308 }
 309
 310 static struct mlxsw_sp_rif *
 311 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 312                          const struct net_device *dev);
 313
 314 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 315
 316 struct mlxsw_sp_prefix_usage {
 317         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 318 };
 319
 320 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 321         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 322
 323 static bool
 324 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
 325                              struct mlxsw_sp_prefix_usage *prefix_usage2)
 326 {
 327         unsigned char prefix;
 328
 329         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
 330                 if (!test_bit(prefix, prefix_usage2->b))
 331                         return false;
 332         }
 333         return true;
 334 }
 335
 336 static bool
 337 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 338                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 339 {
 340         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 341 }
 342
 343 static bool
 344 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
 345 {
 346         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
 347
 348         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 349 }
 350
 351 static void
 352 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 353                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 354 {
 355         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 356 }
 357
 358 static void
 359 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 360                           unsigned char prefix_len)
 361 {
 362         set_bit(prefix_len, prefix_usage->b);
 363 }
 364
 365 static void
 366 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 367                             unsigned char prefix_len)
 368 {
 369         clear_bit(prefix_len, prefix_usage->b);
 370 }
 371
 372 struct mlxsw_sp_fib_key {
 373         unsigned char addr[sizeof(struct in6_addr)];
 374         unsigned char prefix_len;
 375 };
 376
 377 enum mlxsw_sp_fib_entry_type {
 378         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 379         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 380         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 381 };
 382
 383 struct mlxsw_sp_nexthop_group;
 384 struct mlxsw_sp_fib;
 385
 386 struct mlxsw_sp_fib_node {
 387         struct list_head entry_list;
 388         struct list_head list;
 389         struct rhash_head ht_node;
 390         struct mlxsw_sp_fib *fib;
 391         struct mlxsw_sp_fib_key key;
 392 };
 393
 394 struct mlxsw_sp_fib_entry {
 395         struct list_head list;
 396         struct mlxsw_sp_fib_node *fib_node;
 397         enum mlxsw_sp_fib_entry_type type;
 398         struct list_head nexthop_group_node;
 399         struct mlxsw_sp_nexthop_group *nh_group;
 400 };
 401
 402 struct mlxsw_sp_fib4_entry {
 403         struct mlxsw_sp_fib_entry common;
 404         u32 tb_id;
 405         u32 prio;
 406         u8 tos;
 407         u8 type;
 408 };
 409
 410 struct mlxsw_sp_fib6_entry {
 411         struct mlxsw_sp_fib_entry common;
 412         struct list_head rt6_list;
 413         unsigned int nrt6;
 414 };
 415
 416 struct mlxsw_sp_rt6 {
 417         struct list_head list;
 418         struct rt6_info *rt;
 419 };
 420
 421 enum mlxsw_sp_l3proto {
 422         MLXSW_SP_L3_PROTO_IPV4,
 423         MLXSW_SP_L3_PROTO_IPV6,
 424 };
 425
 426 struct mlxsw_sp_lpm_tree {
 427         u8 id; /* tree ID */
 428         unsigned int ref_count;
 429         enum mlxsw_sp_l3proto proto;
 430         struct mlxsw_sp_prefix_usage prefix_usage;
 431 };
 432
 433 struct mlxsw_sp_fib {
 434         struct rhashtable ht;
 435         struct list_head node_list;
 436         struct mlxsw_sp_vr *vr;
 437         struct mlxsw_sp_lpm_tree *lpm_tree;
 438         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 439         struct mlxsw_sp_prefix_usage prefix_usage;
 440         enum mlxsw_sp_l3proto proto;
 441 };
 442
 443 struct mlxsw_sp_vr {
 444         u16 id; /* virtual router ID */
 445         u32 tb_id; /* kernel fib table id */
 446         unsigned int rif_count;
 447         struct mlxsw_sp_fib *fib4;
 448         struct mlxsw_sp_fib *fib6;
 449 };
 450
 451 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 452
 453 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 454                                                 enum mlxsw_sp_l3proto proto)
 455 {
 456         struct mlxsw_sp_fib *fib;
 457         int err;
 458
 459         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 460         if (!fib)
 461                 return ERR_PTR(-ENOMEM);
 462         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 463         if (err)
 464                 goto err_rhashtable_init;
 465         INIT_LIST_HEAD(&fib->node_list);
 466         fib->proto = proto;
 467         fib->vr = vr;
 468         return fib;
 469
 470 err_rhashtable_init:
 471         kfree(fib);
 472         return ERR_PTR(err);
 473 }
 474
 475 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 476 {
 477         WARN_ON(!list_empty(&fib->node_list));
 478         WARN_ON(fib->lpm_tree);
 479         rhashtable_destroy(&fib->ht);
 480         kfree(fib);
 481 }
 482
 483 static struct mlxsw_sp_lpm_tree *
 484 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 485 {
 486         static struct mlxsw_sp_lpm_tree *lpm_tree;
 487         int i;
 488
 489         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 490                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 491                 if (lpm_tree->ref_count == 0)
 492                         return lpm_tree;
 493         }
 494         return NULL;
 495 }
 496
 497 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 498                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 499 {
 500         char ralta_pl[MLXSW_REG_RALTA_LEN];
 501
 502         mlxsw_reg_ralta_pack(ralta_pl, true,
 503                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 504                              lpm_tree->id);
 505         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 506 }
 507
 508 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 509                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 510 {
 511         char ralta_pl[MLXSW_REG_RALTA_LEN];
 512
 513         mlxsw_reg_ralta_pack(ralta_pl, false,
 514                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 515                              lpm_tree->id);
 516         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 517 }
 518
 519 static int
 520 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 521                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 522                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 523 {
 524         char ralst_pl[MLXSW_REG_RALST_LEN];
 525         u8 root_bin = 0;
 526         u8 prefix;
 527         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 528
 529         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 530                 root_bin = prefix;
 531
 532         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 533         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 534                 if (prefix == 0)
 535                         continue;
 536                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 537                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 538                 last_prefix = prefix;
 539         }
 540         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 541 }
 542
 543 static struct mlxsw_sp_lpm_tree *
 544 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 545                          struct mlxsw_sp_prefix_usage *prefix_usage,
 546                          enum mlxsw_sp_l3proto proto)
 547 {
 548         struct mlxsw_sp_lpm_tree *lpm_tree;
 549         int err;
 550
 551         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 552         if (!lpm_tree)
 553                 return ERR_PTR(-EBUSY);
 554         lpm_tree->proto = proto;
 555         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 556         if (err)
 557                 return ERR_PTR(err);
 558
 559         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 560                                                 lpm_tree);
 561         if (err)
 562                 goto err_left_struct_set;
 563         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 564                sizeof(lpm_tree->prefix_usage));
 565         return lpm_tree;
 566
 567 err_left_struct_set:
 568         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 569         return ERR_PTR(err);
 570 }
 571
 572 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 573                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 574 {
 575         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 576 }
 577
 578 static struct mlxsw_sp_lpm_tree *
 579 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 580                       struct mlxsw_sp_prefix_usage *prefix_usage,
 581                       enum mlxsw_sp_l3proto proto)
 582 {
 583         struct mlxsw_sp_lpm_tree *lpm_tree;
 584         int i;
 585
 586         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 587                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 588                 if (lpm_tree->ref_count != 0 &&
 589                     lpm_tree->proto == proto &&
 590                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 591                                              prefix_usage))
 592                         goto inc_ref_count;
 593         }
 594         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 595                                             proto);
 596         if (IS_ERR(lpm_tree))
 597                 return lpm_tree;
 598
 599 inc_ref_count:
 600         lpm_tree->ref_count++;
 601         return lpm_tree;
 602 }
 603
 604 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 605                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 606 {
 607         if (--lpm_tree->ref_count == 0)
 608                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 609 }
 610
 611 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 612
 613 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 614 {
 615         struct mlxsw_sp_lpm_tree *lpm_tree;
 616         u64 max_trees;
 617         int i;
 618
 619         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 620                 return -EIO;
 621
 622         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 623         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 624         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 625                                              sizeof(struct mlxsw_sp_lpm_tree),
 626                                              GFP_KERNEL);
 627         if (!mlxsw_sp->router->lpm.trees)
 628                 return -ENOMEM;
 629
 630         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 631                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 632                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 633         }
 634
 635         return 0;
 636 }
 637
 638 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 639 {
 640         kfree(mlxsw_sp->router->lpm.trees);
 641 }
 642
 643 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 644 {
 645         return !!vr->fib4 || !!vr->fib6;
 646 }
 647
 648 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 649 {
 650         struct mlxsw_sp_vr *vr;
 651         int i;
 652
 653         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 654                 vr = &mlxsw_sp->router->vrs[i];
 655                 if (!mlxsw_sp_vr_is_used(vr))
 656                         return vr;
 657         }
 658         return NULL;
 659 }
 660
 661 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 662                                      const struct mlxsw_sp_fib *fib)
 663 {
 664         char raltb_pl[MLXSW_REG_RALTB_LEN];
 665
 666         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 667                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 668                              fib->lpm_tree->id);
 669         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 670 }
 671
 672 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 673                                        const struct mlxsw_sp_fib *fib)
 674 {
 675         char raltb_pl[MLXSW_REG_RALTB_LEN];
 676
 677         /* Bind to tree 0 which is default */
 678         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 679                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 680         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 681 }
 682
 683 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 684 {
 685         /* For our purpose, squash main and local table into one */
 686         if (tb_id == RT_TABLE_LOCAL)
 687                 tb_id = RT_TABLE_MAIN;
 688         return tb_id;
 689 }
 690
 691 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 692                                             u32 tb_id)
 693 {
 694         struct mlxsw_sp_vr *vr;
 695         int i;
 696
 697         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 698
 699         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 700                 vr = &mlxsw_sp->router->vrs[i];
 701                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 702                         return vr;
 703         }
 704         return NULL;
 705 }
 706
 707 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 708                                             enum mlxsw_sp_l3proto proto)
 709 {
 710         switch (proto) {
 711         case MLXSW_SP_L3_PROTO_IPV4:
 712                 return vr->fib4;
 713         case MLXSW_SP_L3_PROTO_IPV6:
 714                 return vr->fib6;
 715         }
 716         return NULL;
 717 }
 718
 719 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 720                                               u32 tb_id)
 721 {
 722         struct mlxsw_sp_vr *vr;
 723         int err;
 724
 725         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 726         if (!vr)
 727                 return ERR_PTR(-EBUSY);
 728         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 729         if (IS_ERR(vr->fib4))
 730                 return ERR_CAST(vr->fib4);
 731         vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
 732         if (IS_ERR(vr->fib6)) {
 733                 err = PTR_ERR(vr->fib6);
 734                 goto err_fib6_create;
 735         }
 736         vr->tb_id = tb_id;
 737         return vr;
 738
 739 err_fib6_create:
 740         mlxsw_sp_fib_destroy(vr->fib4);
 741         vr->fib4 = NULL;
 742         return ERR_PTR(err);
 743 }
 744
 745 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 746 {
 747         mlxsw_sp_fib_destroy(vr->fib6);
 748         vr->fib6 = NULL;
 749         mlxsw_sp_fib_destroy(vr->fib4);
 750         vr->fib4 = NULL;
 751 }
 752
 753 static int
 754 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
 755                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 756 {
 757         struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
 758         struct mlxsw_sp_lpm_tree *new_tree;
 759         int err;
 760
 761         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
 762                 return 0;
 763
 764         new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 765                                          fib->proto);
 766         if (IS_ERR(new_tree)) {
 767                 /* We failed to get a tree according to the required
 768                  * prefix usage. However, the current tree might be still good
 769                  * for us if our requirement is subset of the prefixes used
 770                  * in the tree.
 771                  */
 772                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 773                                                  &lpm_tree->prefix_usage))
 774                         return 0;
 775                 return PTR_ERR(new_tree);
 776         }
 777
 778         /* Prevent packet loss by overwriting existing binding */
 779         fib->lpm_tree = new_tree;
 780         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
 781         if (err)
 782                 goto err_tree_bind;
 783         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 784
 785         return 0;
 786
 787 err_tree_bind:
 788         fib->lpm_tree = lpm_tree;
 789         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 790         return err;
 791 }
 792
 793 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 794 {
 795         struct mlxsw_sp_vr *vr;
 796
 797         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 798         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 799         if (!vr)
 800                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 801         return vr;
 802 }
 803
 804 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 805 {
 806         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 807             list_empty(&vr->fib6->node_list))
 808                 mlxsw_sp_vr_destroy(vr);
 809 }
 810
 811 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 812 {
 813         struct mlxsw_sp_vr *vr;
 814         u64 max_vrs;
 815         int i;
 816
 817         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 818                 return -EIO;
 819
 820         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 821         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 822                                         GFP_KERNEL);
 823         if (!mlxsw_sp->router->vrs)
 824                 return -ENOMEM;
 825
 826         for (i = 0; i < max_vrs; i++) {
 827                 vr = &mlxsw_sp->router->vrs[i];
 828                 vr->id = i;
 829         }
 830
 831         return 0;
 832 }
 833
 834 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 835
 836 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 837 {
 838         /* At this stage we're guaranteed not to have new incoming
 839          * FIB notifications and the work queue is free from FIBs
 840          * sitting on top of mlxsw netdevs. However, we can still
 841          * have other FIBs queued. Flush the queue before flushing
 842          * the device's tables. No need for locks, as we're the only
 843          * writer.
 844          */
 845         mlxsw_core_flush_owq();
 846         mlxsw_sp_router_fib_flush(mlxsw_sp);
 847         kfree(mlxsw_sp->router->vrs);
 848 }
 849
 850 struct mlxsw_sp_neigh_key {
 851         struct neighbour *n;
 852 };
 853
 854 struct mlxsw_sp_neigh_entry {
 855         struct list_head rif_list_node;
 856         struct rhash_head ht_node;
 857         struct mlxsw_sp_neigh_key key;
 858         u16 rif;
 859         bool connected;
 860         unsigned char ha[ETH_ALEN];
 861         struct list_head nexthop_list; /* list of nexthops using
 862                                         * this neigh entry
 863                                         */
 864         struct list_head nexthop_neighs_list_node;
 865 };
 866
 867 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 868         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 869         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 870         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 871 };
 872
 873 static struct mlxsw_sp_neigh_entry *
 874 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 875                            u16 rif)
 876 {
 877         struct mlxsw_sp_neigh_entry *neigh_entry;
 878
 879         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
 880         if (!neigh_entry)
 881                 return NULL;
 882
 883         neigh_entry->key.n = n;
 884         neigh_entry->rif = rif;
 885         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 886
 887         return neigh_entry;
 888 }
 889
 890 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 891 {
 892         kfree(neigh_entry);
 893 }
 894
 895 static int
 896 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 897                             struct mlxsw_sp_neigh_entry *neigh_entry)
 898 {
 899         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
 900                                       &neigh_entry->ht_node,
 901                                       mlxsw_sp_neigh_ht_params);
 902 }
 903
 904 static void
 905 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 906                             struct mlxsw_sp_neigh_entry *neigh_entry)
 907 {
 908         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
 909                                &neigh_entry->ht_node,
 910                                mlxsw_sp_neigh_ht_params);
 911 }
 912
 913 static struct mlxsw_sp_neigh_entry *
 914 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 915 {
 916         struct mlxsw_sp_neigh_entry *neigh_entry;
 917         struct mlxsw_sp_rif *rif;
 918         int err;
 919
 920         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 921         if (!rif)
 922                 return ERR_PTR(-EINVAL);
 923
 924         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
 925         if (!neigh_entry)
 926                 return ERR_PTR(-ENOMEM);
 927
 928         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 929         if (err)
 930                 goto err_neigh_entry_insert;
 931
 932         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
 933
 934         return neigh_entry;
 935
 936 err_neigh_entry_insert:
 937         mlxsw_sp_neigh_entry_free(neigh_entry);
 938         return ERR_PTR(err);
 939 }
 940
 941 static void
 942 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 943                              struct mlxsw_sp_neigh_entry *neigh_entry)
 944 {
 945         list_del(&neigh_entry->rif_list_node);
 946         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 947         mlxsw_sp_neigh_entry_free(neigh_entry);
 948 }
 949
 950 static struct mlxsw_sp_neigh_entry *
 951 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 952 {
 953         struct mlxsw_sp_neigh_key key;
 954
 955         key.n = n;
 956         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
 957                                       &key, mlxsw_sp_neigh_ht_params);
 958 }
 959
 960 static void
 961 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 962 {
 963         unsigned long interval;
 964
 965 #if IS_ENABLED(CONFIG_IPV6)
 966         interval = min_t(unsigned long,
 967                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
 968                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
 969 #else
 970         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 971 #endif
 972         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
 973 }
 974
 975 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 976                                                    char *rauhtd_pl,
 977                                                    int ent_index)
 978 {
 979         struct net_device *dev;
 980         struct neighbour *n;
 981         __be32 dipn;
 982         u32 dip;
 983         u16 rif;
 984
 985         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 986
 987         if (!mlxsw_sp->router->rifs[rif]) {
 988                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 989                 return;
 990         }
 991
 992         dipn = htonl(dip);
 993         dev = mlxsw_sp->router->rifs[rif]->dev;
 994         n = neigh_lookup(&arp_tbl, &dipn, dev);
 995         if (!n) {
 996                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
 997                            &dip);
 998                 return;
 999         }
1000
1001         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
1002         neigh_event_send(n, NULL);
1003         neigh_release(n);
1004 }
1005
1006 #if IS_ENABLED(IPV6)
1007 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1008                                                    char *rauhtd_pl,
1009                                                    int rec_index)
1010 {
1011         struct net_device *dev;
1012         struct neighbour *n;
1013         struct in6_addr dip;
1014         u16 rif;
1015
1016         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
1017                                          (char *) &dip);
1018
1019         if (!mlxsw_sp->router->rifs[rif]) {
1020                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1021                 return;
1022         }
1023
1024         dev = mlxsw_sp->router->rifs[rif]->dev;
1025         n = neigh_lookup(&nd_tbl, &dip, dev);
1026         if (!n) {
1027                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
1028                            &dip);
1029                 return;
1030         }
1031
1032         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1033         neigh_event_send(n, NULL);
1034         neigh_release(n);
1035 }
1036 #else
1037 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1038                                                    char *rauhtd_pl,
1039                                                    int rec_index)
1040 {
1041 }
1042 #endif
1043
1044 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1045                                                    char *rauhtd_pl,
1046                                                    int rec_index)
1047 {
1048         u8 num_entries;
1049         int i;
1050
1051         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1052                                                                 rec_index);
1053         /* Hardware starts counting at 0, so add 1. */
1054         num_entries++;
1055
1056         /* Each record consists of several neighbour entries. */
1057         for (i = 0; i < num_entries; i++) {
1058                 int ent_index;
1059
1060                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1061                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1062                                                        ent_index);
1063         }
1064
1065 }
1066
1067 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1068                                                    char *rauhtd_pl,
1069                                                    int rec_index)
1070 {
1071         /* One record contains one entry. */
1072         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1073                                                rec_index);
1074 }
1075
1076 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1077                                               char *rauhtd_pl, int rec_index)
1078 {
1079         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1080         case MLXSW_REG_RAUHTD_TYPE_IPV4:
1081                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1082                                                        rec_index);
1083                 break;
1084         case MLXSW_REG_RAUHTD_TYPE_IPV6:
1085                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1086                                                        rec_index);
1087                 break;
1088         }
1089 }
1090
1091 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1092 {
1093         u8 num_rec, last_rec_index, num_entries;
1094
1095         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1096         last_rec_index = num_rec - 1;
1097
1098         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1099                 return false;
1100         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1101             MLXSW_REG_RAUHTD_TYPE_IPV6)
1102                 return true;
1103
1104         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1105                                                                 last_rec_index);
1106         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1107                 return true;
1108         return false;
1109 }
1110
1111 static int
1112 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1113                                        char *rauhtd_pl,
1114                                        enum mlxsw_reg_rauhtd_type type)
1115 {
1116         int i, num_rec;
1117         int err;
1118
1119         /* Make sure the neighbour's netdev isn't removed in the
1120          * process.
1121          */
1122         rtnl_lock();
1123         do {
1124                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
1125                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1126                                       rauhtd_pl);
1127                 if (err) {
1128                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1129                         break;
1130                 }
1131                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1132                 for (i = 0; i < num_rec; i++)
1133                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1134                                                           i);
1135         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1136         rtnl_unlock();
1137
1138         return err;
1139 }
1140
1141 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1142 {
1143         enum mlxsw_reg_rauhtd_type type;
1144         char *rauhtd_pl;
1145         int err;
1146
1147         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1148         if (!rauhtd_pl)
1149                 return -ENOMEM;
1150
1151         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1152         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1153         if (err)
1154                 goto out;
1155
1156         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1157         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1158 out:
1159         kfree(rauhtd_pl);
1160         return err;
1161 }
1162
1163 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1164 {
1165         struct mlxsw_sp_neigh_entry *neigh_entry;
1166
1167         /* Take RTNL mutex here to prevent lists from changes */
1168         rtnl_lock();
1169         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1170                             nexthop_neighs_list_node)
1171                 /* If this neigh have nexthops, make the kernel think this neigh
1172                  * is active regardless of the traffic.
1173                  */
1174                 neigh_event_send(neigh_entry->key.n, NULL);
1175         rtnl_unlock();
1176 }
1177
1178 static void
1179 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1180 {
1181         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1182
1183         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1184                                msecs_to_jiffies(interval));
1185 }
1186
1187 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1188 {
1189         struct mlxsw_sp_router *router;
1190         int err;
1191
1192         router = container_of(work, struct mlxsw_sp_router,
1193                               neighs_update.dw.work);
1194         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1195         if (err)
1196                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1197
1198         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1199
1200         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1201 }
1202
1203 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1204 {
1205         struct mlxsw_sp_neigh_entry *neigh_entry;
1206         struct mlxsw_sp_router *router;
1207
1208         router = container_of(work, struct mlxsw_sp_router,
1209                               nexthop_probe_dw.work);
1210         /* Iterate over nexthop neighbours, find those who are unresolved and
1211          * send arp on them. This solves the chicken-egg problem when
1212          * the nexthop wouldn't get offloaded until the neighbor is resolved
1213          * but it wouldn't get resolved ever in case traffic is flowing in HW
1214          * using different nexthop.
1215          *
1216          * Take RTNL mutex here to prevent lists from changes.
1217          */
1218         rtnl_lock();
1219         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1220                             nexthop_neighs_list_node)
1221                 if (!neigh_entry->connected)
1222                         neigh_event_send(neigh_entry->key.n, NULL);
1223         rtnl_unlock();
1224
1225         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1226                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1227 }
1228
1229 static void
1230 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1231                               struct mlxsw_sp_neigh_entry *neigh_entry,
1232                               bool removing);
1233
1234 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1235 {
1236         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1237                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1238 }
1239
1240 static void
1241 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1242                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1243                                 enum mlxsw_reg_rauht_op op)
1244 {
1245         struct neighbour *n = neigh_entry->key.n;
1246         u32 dip = ntohl(*((__be32 *) n->primary_key));
1247         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1248
1249         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1250                               dip);
1251         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1252 }
1253
1254 static void
1255 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1256                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1257                                 enum mlxsw_reg_rauht_op op)
1258 {
1259         struct neighbour *n = neigh_entry->key.n;
1260         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1261         const char *dip = n->primary_key;
1262
1263         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1264                               dip);
1265         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1266 }
1267
1268 static bool mlxsw_sp_neigh_ipv6_ignore(struct neighbour *n)
1269 {
1270         /* Packets with a link-local destination address are trapped
1271          * after LPM lookup and never reach the neighbour table, so
1272          * there is no need to program such neighbours to the device.
1273          */
1274         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1275             IPV6_ADDR_LINKLOCAL)
1276                 return true;
1277         return false;
1278 }
1279
1280 static void
1281 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1282                             struct mlxsw_sp_neigh_entry *neigh_entry,
1283                             bool adding)
1284 {
1285         if (!adding && !neigh_entry->connected)
1286                 return;
1287         neigh_entry->connected = adding;
1288         if (neigh_entry->key.n->tbl->family == AF_INET) {
1289                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1290                                                 mlxsw_sp_rauht_op(adding));
1291         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
1292                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry->key.n))
1293                         return;
1294                 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1295                                                 mlxsw_sp_rauht_op(adding));
1296         } else {
1297                 WARN_ON_ONCE(1);
1298         }
1299 }
1300
1301 struct mlxsw_sp_neigh_event_work {
1302         struct work_struct work;
1303         struct mlxsw_sp *mlxsw_sp;
1304         struct neighbour *n;
1305 };
1306
1307 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1308 {
1309         struct mlxsw_sp_neigh_event_work *neigh_work =
1310                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1311         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1312         struct mlxsw_sp_neigh_entry *neigh_entry;
1313         struct neighbour *n = neigh_work->n;
1314         unsigned char ha[ETH_ALEN];
1315         bool entry_connected;
1316         u8 nud_state, dead;
1317
1318         /* If these parameters are changed after we release the lock,
1319          * then we are guaranteed to receive another event letting us
1320          * know about it.
1321          */
1322         read_lock_bh(&n->lock);
1323         memcpy(ha, n->ha, ETH_ALEN);
1324         nud_state = n->nud_state;
1325         dead = n->dead;
1326         read_unlock_bh(&n->lock);
1327
1328         rtnl_lock();
1329         entry_connected = nud_state & NUD_VALID && !dead;
1330         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1331         if (!entry_connected && !neigh_entry)
1332                 goto out;
1333         if (!neigh_entry) {
1334                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1335                 if (IS_ERR(neigh_entry))
1336                         goto out;
1337         }
1338
1339         memcpy(neigh_entry->ha, ha, ETH_ALEN);
1340         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1341         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1342
1343         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1344                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1345
1346 out:
1347         rtnl_unlock();
1348         neigh_release(n);
1349         kfree(neigh_work);
1350 }
1351
1352 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1353                                    unsigned long event, void *ptr)
1354 {
1355         struct mlxsw_sp_neigh_event_work *neigh_work;
1356         struct mlxsw_sp_port *mlxsw_sp_port;
1357         struct mlxsw_sp *mlxsw_sp;
1358         unsigned long interval;
1359         struct neigh_parms *p;
1360         struct neighbour *n;
1361
1362         switch (event) {
1363         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1364                 p = ptr;
1365
1366                 /* We don't care about changes in the default table. */
1367                 if (!p->dev || (p->tbl->family != AF_INET &&
1368                                 p->tbl->family != AF_INET6))
1369                         return NOTIFY_DONE;
1370
1371                 /* We are in atomic context and can't take RTNL mutex,
1372                  * so use RCU variant to walk the device chain.
1373                  */
1374                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1375                 if (!mlxsw_sp_port)
1376                         return NOTIFY_DONE;
1377
1378                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1379                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1380                 mlxsw_sp->router->neighs_update.interval = interval;
1381
1382                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1383                 break;
1384         case NETEVENT_NEIGH_UPDATE:
1385                 n = ptr;
1386
1387                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
1388                         return NOTIFY_DONE;
1389
1390                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1391                 if (!mlxsw_sp_port)
1392                         return NOTIFY_DONE;
1393
1394                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1395                 if (!neigh_work) {
1396                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1397                         return NOTIFY_BAD;
1398                 }
1399
1400                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1401                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1402                 neigh_work->n = n;
1403
1404                 /* Take a reference to ensure the neighbour won't be
1405                  * destructed until we drop the reference in delayed
1406                  * work.
1407                  */
1408                 neigh_clone(n);
1409                 mlxsw_core_schedule_work(&neigh_work->work);
1410                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1411                 break;
1412         }
1413
1414         return NOTIFY_DONE;
1415 }
1416
1417 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1418 {
1419         int err;
1420
1421         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1422                               &mlxsw_sp_neigh_ht_params);
1423         if (err)
1424                 return err;
1425
1426         /* Initialize the polling interval according to the default
1427          * table.
1428          */
1429         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1430
1431         /* Create the delayed works for the activity_update */
1432         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1433                           mlxsw_sp_router_neighs_update_work);
1434         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1435                           mlxsw_sp_router_probe_unresolved_nexthops);
1436         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1437         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1438         return 0;
1439 }
1440
1441 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1442 {
1443         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1444         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1445         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
1446 }
1447
1448 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1449                                          struct mlxsw_sp_rif *rif)
1450 {
1451         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1452
1453         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1454                                  rif_list_node) {
1455                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
1456                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1457         }
1458 }
1459
1460 struct mlxsw_sp_nexthop_key {
1461         struct fib_nh *fib_nh;
1462 };
1463
1464 struct mlxsw_sp_nexthop {
1465         struct list_head neigh_list_node; /* member of neigh entry list */
1466         struct list_head rif_list_node;
1467         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1468                                                 * this belongs to
1469                                                 */
1470         struct rhash_head ht_node;
1471         struct mlxsw_sp_nexthop_key key;
1472         unsigned char gw_addr[sizeof(struct in6_addr)];
1473         struct mlxsw_sp_rif *rif;
1474         u8 should_offload:1, /* set indicates this neigh is connected and
1475                               * should be put to KVD linear area of this group.
1476                               */
1477            offloaded:1, /* set in case the neigh is actually put into
1478                          * KVD linear area of this group.
1479                          */
1480            update:1; /* set indicates that MAC of this neigh should be
1481                       * updated in HW
1482                       */
1483         struct mlxsw_sp_neigh_entry *neigh_entry;
1484 };
1485
1486 struct mlxsw_sp_nexthop_group_key {
1487         struct fib_info *fi;
1488 };
1489
1490 struct mlxsw_sp_nexthop_group {
1491         struct rhash_head ht_node;
1492         struct list_head fib_list; /* list of fib entries that use this group */
1493         struct neigh_table *neigh_tbl;
1494         struct mlxsw_sp_nexthop_group_key key;
1495         u8 adj_index_valid:1,
1496            gateway:1; /* routes using the group use a gateway */
1497         u32 adj_index;
1498         u16 ecmp_size;
1499         u16 count;
1500         struct mlxsw_sp_nexthop nexthops[0];
1501 #define nh_rif  nexthops[0].rif
1502 };
1503
1504 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1505         .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1506         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1507         .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1508 };
1509
1510 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1511                                          struct mlxsw_sp_nexthop_group *nh_grp)
1512 {
1513         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
1514                                       &nh_grp->ht_node,
1515                                       mlxsw_sp_nexthop_group_ht_params);
1516 }
1517
1518 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1519                                           struct mlxsw_sp_nexthop_group *nh_grp)
1520 {
1521         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
1522                                &nh_grp->ht_node,
1523                                mlxsw_sp_nexthop_group_ht_params);
1524 }
1525
1526 static struct mlxsw_sp_nexthop_group *
1527 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1528                               struct mlxsw_sp_nexthop_group_key key)
1529 {
1530         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, &key,
1531                                       mlxsw_sp_nexthop_group_ht_params);
1532 }
1533
1534 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1535         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1536         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1537         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1538 };
1539
1540 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1541                                    struct mlxsw_sp_nexthop *nh)
1542 {
1543         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
1544                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1545 }
1546
1547 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1548                                     struct mlxsw_sp_nexthop *nh)
1549 {
1550         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
1551                                mlxsw_sp_nexthop_ht_params);
1552 }
1553
1554 static struct mlxsw_sp_nexthop *
1555 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1556                         struct mlxsw_sp_nexthop_key key)
1557 {
1558         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
1559                                       mlxsw_sp_nexthop_ht_params);
1560 }
1561
1562 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1563                                              const struct mlxsw_sp_fib *fib,
1564                                              u32 adj_index, u16 ecmp_size,
1565                                              u32 new_adj_index,
1566                                              u16 new_ecmp_size)
1567 {
1568         char raleu_pl[MLXSW_REG_RALEU_LEN];
1569
1570         mlxsw_reg_raleu_pack(raleu_pl,
1571                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
1572                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
1573                              new_ecmp_size);
1574         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1575 }
1576
1577 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1578                                           struct mlxsw_sp_nexthop_group *nh_grp,
1579                                           u32 old_adj_index, u16 old_ecmp_size)
1580 {
1581         struct mlxsw_sp_fib_entry *fib_entry;
1582         struct mlxsw_sp_fib *fib = NULL;
1583         int err;
1584
1585         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1586                 if (fib == fib_entry->fib_node->fib)
1587                         continue;
1588                 fib = fib_entry->fib_node->fib;
1589                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1590                                                         old_adj_index,
1591                                                         old_ecmp_size,
1592                                                         nh_grp->adj_index,
1593                                                         nh_grp->ecmp_size);
1594                 if (err)
1595                         return err;
1596         }
1597         return 0;
1598 }
1599
1600 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1601                                        struct mlxsw_sp_nexthop *nh)
1602 {
1603         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1604         char ratr_pl[MLXSW_REG_RATR_LEN];
1605
1606         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1607                             true, adj_index, neigh_entry->rif);
1608         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1609         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1610 }
1611
1612 static int
1613 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1614                                   struct mlxsw_sp_nexthop_group *nh_grp,
1615                                   bool reallocate)
1616 {
1617         u32 adj_index = nh_grp->adj_index; /* base */
1618         struct mlxsw_sp_nexthop *nh;
1619         int i;
1620         int err;
1621
1622         for (i = 0; i < nh_grp->count; i++) {
1623                 nh = &nh_grp->nexthops[i];
1624
1625                 if (!nh->should_offload) {
1626                         nh->offloaded = 0;
1627                         continue;
1628                 }
1629
1630                 if (nh->update || reallocate) {
1631                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1632                                                           adj_index, nh);
1633                         if (err)
1634                                 return err;
1635                         nh->update = 0;
1636                         nh->offloaded = 1;
1637                 }
1638                 adj_index++;
1639         }
1640         return 0;
1641 }
1642
1643 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1644                                      struct mlxsw_sp_fib_entry *fib_entry);
1645
1646 static bool
1647 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
1648                                  const struct mlxsw_sp_fib_entry *fib_entry);
1649
1650 static int
1651 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1652                                     struct mlxsw_sp_nexthop_group *nh_grp)
1653 {
1654         struct mlxsw_sp_fib_entry *fib_entry;
1655         int err;
1656
1657         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1658                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
1659                                                       fib_entry))
1660                         continue;
1661                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1662                 if (err)
1663                         return err;
1664         }
1665         return 0;
1666 }
1667
1668 static void
1669 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1670                                    enum mlxsw_reg_ralue_op op, int err);
1671
1672 static void
1673 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
1674 {
1675         enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
1676         struct mlxsw_sp_fib_entry *fib_entry;
1677
1678         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1679                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
1680                                                       fib_entry))
1681                         continue;
1682                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
1683         }
1684 }
1685
1686 static void
1687 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1688                                struct mlxsw_sp_nexthop_group *nh_grp)
1689 {
1690         struct mlxsw_sp_nexthop *nh;
1691         bool offload_change = false;
1692         u32 adj_index;
1693         u16 ecmp_size = 0;
1694         bool old_adj_index_valid;
1695         u32 old_adj_index;
1696         u16 old_ecmp_size;
1697         int i;
1698         int err;
1699
1700         if (!nh_grp->gateway) {
1701                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1702                 return;
1703         }
1704
1705         for (i = 0; i < nh_grp->count; i++) {
1706                 nh = &nh_grp->nexthops[i];
1707
1708                 if (nh->should_offload != nh->offloaded) {
1709                         offload_change = true;
1710                         if (nh->should_offload)
1711                                 nh->update = 1;
1712                 }
1713                 if (nh->should_offload)
1714                         ecmp_size++;
1715         }
1716         if (!offload_change) {
1717                 /* Nothing was added or removed, so no need to reallocate. Just
1718                  * update MAC on existing adjacency indexes.
1719                  */
1720                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1721                                                         false);
1722                 if (err) {
1723                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1724                         goto set_trap;
1725                 }
1726                 return;
1727         }
1728         if (!ecmp_size)
1729                 /* No neigh of this group is connected so we just set
1730                  * the trap and let everthing flow through kernel.
1731                  */
1732                 goto set_trap;
1733
1734         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
1735         if (err) {
1736                 /* We ran out of KVD linear space, just set the
1737                  * trap and let everything flow through kernel.
1738                  */
1739                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1740                 goto set_trap;
1741         }
1742         old_adj_index_valid = nh_grp->adj_index_valid;
1743         old_adj_index = nh_grp->adj_index;
1744         old_ecmp_size = nh_grp->ecmp_size;
1745         nh_grp->adj_index_valid = 1;
1746         nh_grp->adj_index = adj_index;
1747         nh_grp->ecmp_size = ecmp_size;
1748         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1749         if (err) {
1750                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1751                 goto set_trap;
1752         }
1753
1754         if (!old_adj_index_valid) {
1755                 /* The trap was set for fib entries, so we have to call
1756                  * fib entry update to unset it and use adjacency index.
1757                  */
1758                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1759                 if (err) {
1760                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1761                         goto set_trap;
1762                 }
1763                 return;
1764         }
1765
1766         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1767                                              old_adj_index, old_ecmp_size);
1768         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1769         if (err) {
1770                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1771                 goto set_trap;
1772         }
1773
1774         /* Offload state within the group changed, so update the flags. */
1775         mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
1776
1777         return;
1778
1779 set_trap:
1780         old_adj_index_valid = nh_grp->adj_index_valid;
1781         nh_grp->adj_index_valid = 0;
1782         for (i = 0; i < nh_grp->count; i++) {
1783                 nh = &nh_grp->nexthops[i];
1784                 nh->offloaded = 0;
1785         }
1786         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1787         if (err)
1788                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1789         if (old_adj_index_valid)
1790                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1791 }
1792
1793 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1794                                             bool removing)
1795 {
1796         if (!removing)
1797                 nh->should_offload = 1;
1798         else if (nh->offloaded)
1799                 nh->should_offload = 0;
1800         nh->update = 1;
1801 }
1802
1803 static void
1804 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1805                               struct mlxsw_sp_neigh_entry *neigh_entry,
1806                               bool removing)
1807 {
1808         struct mlxsw_sp_nexthop *nh;
1809
1810         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1811                             neigh_list_node) {
1812                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1813                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1814         }
1815 }
1816
1817 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1818                                       struct mlxsw_sp_rif *rif)
1819 {
1820         if (nh->rif)
1821                 return;
1822
1823         nh->rif = rif;
1824         list_add(&nh->rif_list_node, &rif->nexthop_list);
1825 }
1826
1827 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1828 {
1829         if (!nh->rif)
1830                 return;
1831
1832         list_del(&nh->rif_list_node);
1833         nh->rif = NULL;
1834 }
1835
1836 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1837                                        struct mlxsw_sp_nexthop *nh)
1838 {
1839         struct mlxsw_sp_neigh_entry *neigh_entry;
1840         struct neighbour *n;
1841         u8 nud_state, dead;
1842         int err;
1843
1844         if (!nh->nh_grp->gateway || nh->neigh_entry)
1845                 return 0;
1846
1847         /* Take a reference of neigh here ensuring that neigh would
1848          * not be destructed before the nexthop entry is finished.
1849          * The reference is taken either in neigh_lookup() or
1850          * in neigh_create() in case n is not found.
1851          */
1852         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
1853         if (!n) {
1854                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
1855                                  nh->rif->dev);
1856                 if (IS_ERR(n))
1857                         return PTR_ERR(n);
1858                 neigh_event_send(n, NULL);
1859         }
1860         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1861         if (!neigh_entry) {
1862                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1863                 if (IS_ERR(neigh_entry)) {
1864                         err = -EINVAL;
1865                         goto err_neigh_entry_create;
1866                 }
1867         }
1868
1869         /* If that is the first nexthop connected to that neigh, add to
1870          * nexthop_neighs_list
1871          */
1872         if (list_empty(&neigh_entry->nexthop_list))
1873                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1874                               &mlxsw_sp->router->nexthop_neighs_list);
1875
1876         nh->neigh_entry = neigh_entry;
1877         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1878         read_lock_bh(&n->lock);
1879         nud_state = n->nud_state;
1880         dead = n->dead;
1881         read_unlock_bh(&n->lock);
1882         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1883
1884         return 0;
1885
1886 err_neigh_entry_create:
1887         neigh_release(n);
1888         return err;
1889 }
1890
1891 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1892                                         struct mlxsw_sp_nexthop *nh)
1893 {
1894         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1895         struct neighbour *n;
1896
1897         if (!neigh_entry)
1898                 return;
1899         n = neigh_entry->key.n;
1900
1901         __mlxsw_sp_nexthop_neigh_update(nh, true);
1902         list_del(&nh->neigh_list_node);
1903         nh->neigh_entry = NULL;
1904
1905         /* If that is the last nexthop connected to that neigh, remove from
1906          * nexthop_neighs_list
1907          */
1908         if (list_empty(&neigh_entry->nexthop_list))
1909                 list_del(&neigh_entry->nexthop_neighs_list_node);
1910
1911         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1912                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1913
1914         neigh_release(n);
1915 }
1916
1917 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
1918                                   struct mlxsw_sp_nexthop_group *nh_grp,
1919                                   struct mlxsw_sp_nexthop *nh,
1920                                   struct fib_nh *fib_nh)
1921 {
1922         struct net_device *dev = fib_nh->nh_dev;
1923         struct in_device *in_dev;
1924         struct mlxsw_sp_rif *rif;
1925         int err;
1926
1927         nh->nh_grp = nh_grp;
1928         nh->key.fib_nh = fib_nh;
1929         memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
1930         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1931         if (err)
1932                 return err;
1933
1934         if (!dev)
1935                 return 0;
1936
1937         in_dev = __in_dev_get_rtnl(dev);
1938         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1939             fib_nh->nh_flags & RTNH_F_LINKDOWN)
1940                 return 0;
1941
1942         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1943         if (!rif)
1944                 return 0;
1945         mlxsw_sp_nexthop_rif_init(nh, rif);
1946
1947         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1948         if (err)
1949                 goto err_nexthop_neigh_init;
1950
1951         return 0;
1952
1953 err_nexthop_neigh_init:
1954         mlxsw_sp_nexthop_rif_fini(nh);
1955         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1956         return err;
1957 }
1958
1959 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
1960                                    struct mlxsw_sp_nexthop *nh)
1961 {
1962         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1963         mlxsw_sp_nexthop_rif_fini(nh);
1964         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1965 }
1966
1967 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
1968                                     unsigned long event, struct fib_nh *fib_nh)
1969 {
1970         struct mlxsw_sp_nexthop_key key;
1971         struct mlxsw_sp_nexthop *nh;
1972         struct mlxsw_sp_rif *rif;
1973
1974         if (mlxsw_sp->router->aborted)
1975                 return;
1976
1977         key.fib_nh = fib_nh;
1978         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1979         if (WARN_ON_ONCE(!nh))
1980                 return;
1981
1982         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1983         if (!rif)
1984                 return;
1985
1986         switch (event) {
1987         case FIB_EVENT_NH_ADD:
1988                 mlxsw_sp_nexthop_rif_init(nh, rif);
1989                 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1990                 break;
1991         case FIB_EVENT_NH_DEL:
1992                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1993                 mlxsw_sp_nexthop_rif_fini(nh);
1994                 break;
1995         }
1996
1997         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1998 }
1999
2000 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2001                                            struct mlxsw_sp_rif *rif)
2002 {
2003         struct mlxsw_sp_nexthop *nh, *tmp;
2004
2005         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
2006                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2007                 mlxsw_sp_nexthop_rif_fini(nh);
2008                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2009         }
2010 }
2011
2012 static struct mlxsw_sp_nexthop_group *
2013 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
2014 {
2015         struct mlxsw_sp_nexthop_group *nh_grp;
2016         struct mlxsw_sp_nexthop *nh;
2017         struct fib_nh *fib_nh;
2018         size_t alloc_size;
2019         int i;
2020         int err;
2021
2022         alloc_size = sizeof(*nh_grp) +
2023                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
2024         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
2025         if (!nh_grp)
2026                 return ERR_PTR(-ENOMEM);
2027         INIT_LIST_HEAD(&nh_grp->fib_list);
2028         nh_grp->neigh_tbl = &arp_tbl;
2029
2030         nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
2031         nh_grp->count = fi->fib_nhs;
2032         nh_grp->key.fi = fi;
2033         fib_info_hold(fi);
2034         for (i = 0; i < nh_grp->count; i++) {
2035                 nh = &nh_grp->nexthops[i];
2036                 fib_nh = &fi->fib_nh[i];
2037                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
2038                 if (err)
2039                         goto err_nexthop4_init;
2040         }
2041         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
2042         if (err)
2043                 goto err_nexthop_group_insert;
2044         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2045         return nh_grp;
2046
2047 err_nexthop_group_insert:
2048 err_nexthop4_init:
2049         for (i--; i >= 0; i--) {
2050                 nh = &nh_grp->nexthops[i];
2051                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2052         }
2053         fib_info_put(nh_grp->key.fi);
2054         kfree(nh_grp);
2055         return ERR_PTR(err);
2056 }
2057
2058 static void
2059 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
2060                                 struct mlxsw_sp_nexthop_group *nh_grp)
2061 {
2062         struct mlxsw_sp_nexthop *nh;
2063         int i;
2064
2065         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
2066         for (i = 0; i < nh_grp->count; i++) {
2067                 nh = &nh_grp->nexthops[i];
2068                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2069         }
2070         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2071         WARN_ON_ONCE(nh_grp->adj_index_valid);
2072         fib_info_put(nh_grp->key.fi);
2073         kfree(nh_grp);
2074 }
2075
2076 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
2077                                        struct mlxsw_sp_fib_entry *fib_entry,
2078                                        struct fib_info *fi)
2079 {
2080         struct mlxsw_sp_nexthop_group_key key;
2081         struct mlxsw_sp_nexthop_group *nh_grp;
2082
2083         key.fi = fi;
2084         nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
2085         if (!nh_grp) {
2086                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
2087                 if (IS_ERR(nh_grp))
2088                         return PTR_ERR(nh_grp);
2089         }
2090         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
2091         fib_entry->nh_group = nh_grp;
2092         return 0;
2093 }
2094
2095 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
2096                                         struct mlxsw_sp_fib_entry *fib_entry)
2097 {
2098         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2099
2100         list_del(&fib_entry->nexthop_group_node);
2101         if (!list_empty(&nh_grp->fib_list))
2102                 return;
2103         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
2104 }
2105
2106 static bool
2107 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2108 {
2109         struct mlxsw_sp_fib4_entry *fib4_entry;
2110
2111         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
2112                                   common);
2113         return !fib4_entry->tos;
2114 }
2115
2116 static bool
2117 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2118 {
2119         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
2120
2121         switch (fib_entry->fib_node->fib->proto) {
2122         case MLXSW_SP_L3_PROTO_IPV4:
2123                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
2124                         return false;
2125                 break;
2126         case MLXSW_SP_L3_PROTO_IPV6:
2127                 break;
2128         }
2129
2130         switch (fib_entry->type) {
2131         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2132                 return !!nh_group->adj_index_valid;
2133         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2134                 return !!nh_group->nh_rif;
2135         default:
2136                 return false;
2137         }
2138 }
2139
2140 static struct mlxsw_sp_nexthop *
2141 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
2142                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
2143 {
2144         int i;
2145
2146         for (i = 0; i < nh_grp->count; i++) {
2147                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2148                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2149
2150                 if (nh->rif && nh->rif->dev == rt->dst.dev &&
2151                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
2152                                     &rt->rt6i_gateway))
2153                         return nh;
2154                 continue;
2155         }
2156
2157         return NULL;
2158 }
2159
2160 static void
2161 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2162 {
2163         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2164         int i;
2165
2166         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
2167                 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
2168                 return;
2169         }
2170
2171         for (i = 0; i < nh_grp->count; i++) {
2172                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2173
2174                 if (nh->offloaded)
2175                         nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
2176                 else
2177                         nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
2178         }
2179 }
2180
2181 static void
2182 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2183 {
2184         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2185         int i;
2186
2187         for (i = 0; i < nh_grp->count; i++) {
2188                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2189
2190                 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
2191         }
2192 }
2193
2194 static void
2195 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2196 {
2197         struct mlxsw_sp_fib6_entry *fib6_entry;
2198         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2199
2200         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
2201                                   common);
2202
2203         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
2204                 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
2205                                  list)->rt->rt6i_flags |= RTF_OFFLOAD;
2206                 return;
2207         }
2208
2209         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2210                 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2211                 struct mlxsw_sp_nexthop *nh;
2212
2213                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
2214                 if (nh && nh->offloaded)
2215                         mlxsw_sp_rt6->rt->rt6i_flags |= RTF_OFFLOAD;
2216                 else
2217                         mlxsw_sp_rt6->rt->rt6i_flags &= ~RTF_OFFLOAD;
2218         }
2219 }
2220
2221 static void
2222 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2223 {
2224         struct mlxsw_sp_fib6_entry *fib6_entry;
2225         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2226
2227         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
2228                                   common);
2229         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2230                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2231
2232                 rt->rt6i_flags &= ~RTF_OFFLOAD;
2233         }
2234 }
2235
2236 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2237 {
2238         switch (fib_entry->fib_node->fib->proto) {
2239         case MLXSW_SP_L3_PROTO_IPV4:
2240                 mlxsw_sp_fib4_entry_offload_set(fib_entry);
2241                 break;
2242         case MLXSW_SP_L3_PROTO_IPV6:
2243                 mlxsw_sp_fib6_entry_offload_set(fib_entry);
2244                 break;
2245         }
2246 }
2247
2248 static void
2249 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2250 {
2251         switch (fib_entry->fib_node->fib->proto) {
2252         case MLXSW_SP_L3_PROTO_IPV4:
2253                 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
2254                 break;
2255         case MLXSW_SP_L3_PROTO_IPV6:
2256                 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
2257                 break;
2258         }
2259 }
2260
2261 static void
2262 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2263                                    enum mlxsw_reg_ralue_op op, int err)
2264 {
2265         switch (op) {
2266         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
2267                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
2268         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
2269                 if (err)
2270                         return;
2271                 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
2272                         mlxsw_sp_fib_entry_offload_set(fib_entry);
2273                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
2274                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
2275                 return;
2276         default:
2277                 return;
2278         }
2279 }
2280
2281 static void
2282 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
2283                               const struct mlxsw_sp_fib_entry *fib_entry,
2284                               enum mlxsw_reg_ralue_op op)
2285 {
2286         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2287         enum mlxsw_reg_ralxx_protocol proto;
2288         u32 *p_dip;
2289
2290         proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
2291
2292         switch (fib->proto) {
2293         case MLXSW_SP_L3_PROTO_IPV4:
2294                 p_dip = (u32 *) fib_entry->fib_node->key.addr;
2295                 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
2296                                       fib_entry->fib_node->key.prefix_len,
2297                                       *p_dip);
2298                 break;
2299         case MLXSW_SP_L3_PROTO_IPV6:
2300                 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
2301                                       fib_entry->fib_node->key.prefix_len,
2302                                       fib_entry->fib_node->key.addr);
2303                 break;
2304         }
2305 }
2306
2307 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
2308                                         struct mlxsw_sp_fib_entry *fib_entry,
2309                                         enum mlxsw_reg_ralue_op op)
2310 {
2311         char ralue_pl[MLXSW_REG_RALUE_LEN];
2312         enum mlxsw_reg_ralue_trap_action trap_action;
2313         u16 trap_id = 0;
2314         u32 adjacency_index = 0;
2315         u16 ecmp_size = 0;
2316
2317         /* In case the nexthop group adjacency index is valid, use it
2318          * with provided ECMP size. Otherwise, setup trap and pass
2319          * traffic to kernel.
2320          */
2321         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2322                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2323                 adjacency_index = fib_entry->nh_group->adj_index;
2324                 ecmp_size = fib_entry->nh_group->ecmp_size;
2325         } else {
2326                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2327                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2328         }
2329
2330         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
2331         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2332                                         adjacency_index, ecmp_size);
2333         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2334 }
2335
2336 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
2337                                        struct mlxsw_sp_fib_entry *fib_entry,
2338                                        enum mlxsw_reg_ralue_op op)
2339 {
2340         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
2341         enum mlxsw_reg_ralue_trap_action trap_action;
2342         char ralue_pl[MLXSW_REG_RALUE_LEN];
2343         u16 trap_id = 0;
2344         u16 rif_index = 0;
2345
2346         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2347                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2348                 rif_index = rif->rif_index;
2349         } else {
2350                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2351                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2352         }
2353
2354         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
2355         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
2356                                        rif_index);
2357         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2358 }
2359
2360 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
2361                                       struct mlxsw_sp_fib_entry *fib_entry,
2362                                       enum mlxsw_reg_ralue_op op)
2363 {
2364         char ralue_pl[MLXSW_REG_RALUE_LEN];
2365
2366         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
2367         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2368         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2369 }
2370
2371 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2372                                    struct mlxsw_sp_fib_entry *fib_entry,
2373                                    enum mlxsw_reg_ralue_op op)
2374 {
2375         switch (fib_entry->type) {
2376         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2377                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
2378         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2379                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
2380         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
2381                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
2382         }
2383         return -EINVAL;
2384 }
2385
2386 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2387                                  struct mlxsw_sp_fib_entry *fib_entry,
2388                                  enum mlxsw_reg_ralue_op op)
2389 {
2390         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
2391
2392         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
2393
2394         return err;
2395 }
2396
2397 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2398                                      struct mlxsw_sp_fib_entry *fib_entry)
2399 {
2400         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2401                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
2402 }
2403
2404 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
2405                                   struct mlxsw_sp_fib_entry *fib_entry)
2406 {
2407         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2408                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
2409 }
2410
2411 static int
2412 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
2413                              const struct fib_entry_notifier_info *fen_info,
2414                              struct mlxsw_sp_fib_entry *fib_entry)
2415 {
2416         struct fib_info *fi = fen_info->fi;
2417
2418         switch (fen_info->type) {
2419         case RTN_BROADCAST: /* fall through */
2420         case RTN_LOCAL:
2421                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2422                 return 0;
2423         case RTN_UNREACHABLE: /* fall through */
2424         case RTN_BLACKHOLE: /* fall through */
2425         case RTN_PROHIBIT:
2426                 /* Packets hitting these routes need to be trapped, but
2427                  * can do so with a lower priority than packets directed
2428                  * at the host, so use action type local instead of trap.
2429                  */
2430                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2431                 return 0;
2432         case RTN_UNICAST:
2433                 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
2434                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2435                 else
2436                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
2437                 return 0;
2438         default:
2439                 return -EINVAL;
2440         }
2441 }
2442
2443 static struct mlxsw_sp_fib4_entry *
2444 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
2445                            struct mlxsw_sp_fib_node *fib_node,
2446                            const struct fib_entry_notifier_info *fen_info)
2447 {
2448         struct mlxsw_sp_fib4_entry *fib4_entry;
2449         struct mlxsw_sp_fib_entry *fib_entry;
2450         int err;
2451
2452         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
2453         if (!fib4_entry)
2454                 return ERR_PTR(-ENOMEM);
2455         fib_entry = &fib4_entry->common;
2456
2457         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
2458         if (err)
2459                 goto err_fib4_entry_type_set;
2460
2461         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
2462         if (err)
2463                 goto err_nexthop4_group_get;
2464
2465         fib4_entry->prio = fen_info->fi->fib_priority;
2466         fib4_entry->tb_id = fen_info->tb_id;
2467         fib4_entry->type = fen_info->type;
2468         fib4_entry->tos = fen_info->tos;
2469
2470         fib_entry->fib_node = fib_node;
2471
2472         return fib4_entry;
2473
2474 err_nexthop4_group_get:
2475 err_fib4_entry_type_set:
2476         kfree(fib4_entry);
2477         return ERR_PTR(err);
2478 }
2479
2480 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2481                                         struct mlxsw_sp_fib4_entry *fib4_entry)
2482 {
2483         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
2484         kfree(fib4_entry);
2485 }
2486
2487 static struct mlxsw_sp_fib_node *
2488 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2489                          size_t addr_len, unsigned char prefix_len);
2490
2491 static struct mlxsw_sp_fib4_entry *
2492 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
2493                            const struct fib_entry_notifier_info *fen_info)
2494 {
2495         struct mlxsw_sp_fib4_entry *fib4_entry;
2496         struct mlxsw_sp_fib_node *fib_node;
2497         struct mlxsw_sp_fib *fib;
2498         struct mlxsw_sp_vr *vr;
2499
2500         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
2501         if (!vr)
2502                 return NULL;
2503         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2504
2505         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2506                                             sizeof(fen_info->dst),
2507                                             fen_info->dst_len);
2508         if (!fib_node)
2509                 return NULL;
2510
2511         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
2512                 if (fib4_entry->tb_id == fen_info->tb_id &&
2513                     fib4_entry->tos == fen_info->tos &&
2514                     fib4_entry->type == fen_info->type &&
2515                     fib4_entry->common.nh_group->key.fi == fen_info->fi) {
2516                         return fib4_entry;
2517                 }
2518         }
2519
2520         return NULL;
2521 }
2522
2523 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
2524         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
2525         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
2526         .key_len = sizeof(struct mlxsw_sp_fib_key),
2527         .automatic_shrinking = true,
2528 };
2529
2530 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
2531                                     struct mlxsw_sp_fib_node *fib_node)
2532 {
2533         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
2534                                       mlxsw_sp_fib_ht_params);
2535 }
2536
2537 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
2538                                      struct mlxsw_sp_fib_node *fib_node)
2539 {
2540         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
2541                                mlxsw_sp_fib_ht_params);
2542 }
2543
2544 static struct mlxsw_sp_fib_node *
2545 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2546                          size_t addr_len, unsigned char prefix_len)
2547 {
2548         struct mlxsw_sp_fib_key key;
2549
2550         memset(&key, 0, sizeof(key));
2551         memcpy(key.addr, addr, addr_len);
2552         key.prefix_len = prefix_len;
2553         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2554 }
2555
2556 static struct mlxsw_sp_fib_node *
2557 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
2558                          size_t addr_len, unsigned char prefix_len)
2559 {
2560         struct mlxsw_sp_fib_node *fib_node;
2561
2562         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2563         if (!fib_node)
2564                 return NULL;
2565
2566         INIT_LIST_HEAD(&fib_node->entry_list);
2567         list_add(&fib_node->list, &fib->node_list);
2568         memcpy(fib_node->key.addr, addr, addr_len);
2569         fib_node->key.prefix_len = prefix_len;
2570
2571         return fib_node;
2572 }
2573
2574 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2575 {
2576         list_del(&fib_node->list);
2577         WARN_ON(!list_empty(&fib_node->entry_list));
2578         kfree(fib_node);
2579 }
2580
2581 static bool
2582 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2583                                  const struct mlxsw_sp_fib_entry *fib_entry)
2584 {
2585         return list_first_entry(&fib_node->entry_list,
2586                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
2587 }
2588
2589 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2590 {
2591         unsigned char prefix_len = fib_node->key.prefix_len;
2592         struct mlxsw_sp_fib *fib = fib_node->fib;
2593
2594         if (fib->prefix_ref_count[prefix_len]++ == 0)
2595                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2596 }
2597
2598 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2599 {
2600         unsigned char prefix_len = fib_node->key.prefix_len;
2601         struct mlxsw_sp_fib *fib = fib_node->fib;
2602
2603         if (--fib->prefix_ref_count[prefix_len] == 0)
2604                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2605 }
2606
2607 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2608                                   struct mlxsw_sp_fib_node *fib_node,
2609                                   struct mlxsw_sp_fib *fib)
2610 {
2611         struct mlxsw_sp_prefix_usage req_prefix_usage;
2612         struct mlxsw_sp_lpm_tree *lpm_tree;
2613         int err;
2614
2615         err = mlxsw_sp_fib_node_insert(fib, fib_node);
2616         if (err)
2617                 return err;
2618         fib_node->fib = fib;
2619
2620         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2621         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2622
2623         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2624                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2625                                                  &req_prefix_usage);
2626                 if (err)
2627                         goto err_tree_check;
2628         } else {
2629                 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2630                                                  fib->proto);
2631                 if (IS_ERR(lpm_tree))
2632                         return PTR_ERR(lpm_tree);
2633                 fib->lpm_tree = lpm_tree;
2634                 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2635                 if (err)
2636                         goto err_tree_bind;
2637         }
2638
2639         mlxsw_sp_fib_node_prefix_inc(fib_node);
2640
2641         return 0;
2642
2643 err_tree_bind:
2644         fib->lpm_tree = NULL;
2645         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2646 err_tree_check:
2647         fib_node->fib = NULL;
2648         mlxsw_sp_fib_node_remove(fib, fib_node);
2649         return err;
2650 }
2651
2652 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2653                                    struct mlxsw_sp_fib_node *fib_node)
2654 {
2655         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2656         struct mlxsw_sp_fib *fib = fib_node->fib;
2657
2658         mlxsw_sp_fib_node_prefix_dec(fib_node);
2659
2660         if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2661                 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2662                 fib->lpm_tree = NULL;
2663                 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2664         } else {
2665                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2666         }
2667
2668         fib_node->fib = NULL;
2669         mlxsw_sp_fib_node_remove(fib, fib_node);
2670 }
2671
2672 static struct mlxsw_sp_fib_node *
2673 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
2674                       size_t addr_len, unsigned char prefix_len,
2675                       enum mlxsw_sp_l3proto proto)
2676 {
2677         struct mlxsw_sp_fib_node *fib_node;
2678         struct mlxsw_sp_fib *fib;
2679         struct mlxsw_sp_vr *vr;
2680         int err;
2681
2682         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
2683         if (IS_ERR(vr))
2684                 return ERR_CAST(vr);
2685         fib = mlxsw_sp_vr_fib(vr, proto);
2686
2687         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
2688         if (fib_node)
2689                 return fib_node;
2690
2691         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
2692         if (!fib_node) {
2693                 err = -ENOMEM;
2694                 goto err_fib_node_create;
2695         }
2696
2697         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2698         if (err)
2699                 goto err_fib_node_init;
2700
2701         return fib_node;
2702
2703 err_fib_node_init:
2704         mlxsw_sp_fib_node_destroy(fib_node);
2705 err_fib_node_create:
2706         mlxsw_sp_vr_put(vr);
2707         return ERR_PTR(err);
2708 }
2709
2710 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
2711                                   struct mlxsw_sp_fib_node *fib_node)
2712 {
2713         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2714
2715         if (!list_empty(&fib_node->entry_list))
2716                 return;
2717         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2718         mlxsw_sp_fib_node_destroy(fib_node);
2719         mlxsw_sp_vr_put(vr);
2720 }
2721
2722 static struct mlxsw_sp_fib4_entry *
2723 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2724                               const struct mlxsw_sp_fib4_entry *new4_entry)
2725 {
2726         struct mlxsw_sp_fib4_entry *fib4_entry;
2727
2728         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
2729                 if (fib4_entry->tb_id > new4_entry->tb_id)
2730                         continue;
2731                 if (fib4_entry->tb_id != new4_entry->tb_id)
2732                         break;
2733                 if (fib4_entry->tos > new4_entry->tos)
2734                         continue;
2735                 if (fib4_entry->prio >= new4_entry->prio ||
2736                     fib4_entry->tos < new4_entry->tos)
2737                         return fib4_entry;
2738         }
2739
2740         return NULL;
2741 }
2742
2743 static int
2744 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
2745                                struct mlxsw_sp_fib4_entry *new4_entry)
2746 {
2747         struct mlxsw_sp_fib_node *fib_node;
2748
2749         if (WARN_ON(!fib4_entry))
2750                 return -EINVAL;
2751
2752         fib_node = fib4_entry->common.fib_node;
2753         list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
2754                                  common.list) {
2755                 if (fib4_entry->tb_id != new4_entry->tb_id ||
2756                     fib4_entry->tos != new4_entry->tos ||
2757                     fib4_entry->prio != new4_entry->prio)
2758                         break;
2759         }
2760
2761         list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
2762         return 0;
2763 }
2764
2765 static int
2766 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
2767                                bool replace, bool append)
2768 {
2769         struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
2770         struct mlxsw_sp_fib4_entry *fib4_entry;
2771
2772         fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
2773
2774         if (append)
2775                 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
2776         if (replace && WARN_ON(!fib4_entry))
2777                 return -EINVAL;
2778
2779         /* Insert new entry before replaced one, so that we can later
2780          * remove the second.
2781          */
2782         if (fib4_entry) {
2783                 list_add_tail(&new4_entry->common.list,
2784                               &fib4_entry->common.list);
2785         } else {
2786                 struct mlxsw_sp_fib4_entry *last;
2787
2788                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
2789                         if (new4_entry->tb_id > last->tb_id)
2790                                 break;
2791                         fib4_entry = last;
2792                 }
2793
2794                 if (fib4_entry)
2795                         list_add(&new4_entry->common.list,
2796                                  &fib4_entry->common.list);
2797                 else
2798                         list_add(&new4_entry->common.list,
2799                                  &fib_node->entry_list);
2800         }
2801
2802         return 0;
2803 }
2804
2805 static void
2806 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
2807 {
2808         list_del(&fib4_entry->common.list);
2809 }
2810
2811 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2812                                        struct mlxsw_sp_fib_entry *fib_entry)
2813 {
2814         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2815
2816         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2817                 return 0;
2818
2819         /* To prevent packet loss, overwrite the previously offloaded
2820          * entry.
2821          */
2822         if (!list_is_singular(&fib_node->entry_list)) {
2823                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2824                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2825
2826                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2827         }
2828
2829         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2830 }
2831
2832 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2833                                         struct mlxsw_sp_fib_entry *fib_entry)
2834 {
2835         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2836
2837         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2838                 return;
2839
2840         /* Promote the next entry by overwriting the deleted entry */
2841         if (!list_is_singular(&fib_node->entry_list)) {
2842                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2843                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2844
2845                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2846                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2847                 return;
2848         }
2849
2850         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2851 }
2852
2853 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2854                                          struct mlxsw_sp_fib4_entry *fib4_entry,
2855                                          bool replace, bool append)
2856 {
2857         int err;
2858
2859         err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
2860         if (err)
2861                 return err;
2862
2863         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
2864         if (err)
2865                 goto err_fib_node_entry_add;
2866
2867         return 0;
2868
2869 err_fib_node_entry_add:
2870         mlxsw_sp_fib4_node_list_remove(fib4_entry);
2871         return err;
2872 }
2873
2874 static void
2875 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2876                                 struct mlxsw_sp_fib4_entry *fib4_entry)
2877 {
2878         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
2879         mlxsw_sp_fib4_node_list_remove(fib4_entry);
2880 }
2881
2882 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2883                                         struct mlxsw_sp_fib4_entry *fib4_entry,
2884                                         bool replace)
2885 {
2886         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
2887         struct mlxsw_sp_fib4_entry *replaced;
2888
2889         if (!replace)
2890                 return;
2891
2892         /* We inserted the new entry before replaced one */
2893         replaced = list_next_entry(fib4_entry, common.list);
2894
2895         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2896         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2897         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
2898 }
2899
2900 static int
2901 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2902                          const struct fib_entry_notifier_info *fen_info,
2903                          bool replace, bool append)
2904 {
2905         struct mlxsw_sp_fib4_entry *fib4_entry;
2906         struct mlxsw_sp_fib_node *fib_node;
2907         int err;
2908
2909         if (mlxsw_sp->router->aborted)
2910                 return 0;
2911
2912         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
2913                                          &fen_info->dst, sizeof(fen_info->dst),
2914                                          fen_info->dst_len,
2915                                          MLXSW_SP_L3_PROTO_IPV4);
2916         if (IS_ERR(fib_node)) {
2917                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2918                 return PTR_ERR(fib_node);
2919         }
2920
2921         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2922         if (IS_ERR(fib4_entry)) {
2923                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2924                 err = PTR_ERR(fib4_entry);
2925                 goto err_fib4_entry_create;
2926         }
2927
2928         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
2929                                             append);
2930         if (err) {
2931                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2932                 goto err_fib4_node_entry_link;
2933         }
2934
2935         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
2936
2937         return 0;
2938
2939 err_fib4_node_entry_link:
2940         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
2941 err_fib4_entry_create:
2942         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
2943         return err;
2944 }
2945
2946 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2947                                      struct fib_entry_notifier_info *fen_info)
2948 {
2949         struct mlxsw_sp_fib4_entry *fib4_entry;
2950         struct mlxsw_sp_fib_node *fib_node;
2951
2952         if (mlxsw_sp->router->aborted)
2953                 return;
2954
2955         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2956         if (WARN_ON(!fib4_entry))
2957                 return;
2958         fib_node = fib4_entry->common.fib_node;
2959
2960         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
2961         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
2962         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
2963 }
2964
2965 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
2966 {
2967         /* Packets with link-local destination IP arriving to the router
2968          * are trapped to the CPU, so no need to program specific routes
2969          * for them.
2970          */
2971         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
2972                 return true;
2973
2974         /* Multicast routes aren't supported, so ignore them. Neighbour
2975          * Discovery packets are specifically trapped.
2976          */
2977         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
2978                 return true;
2979
2980         /* Cloned routes are irrelevant in the forwarding path. */
2981         if (rt->rt6i_flags & RTF_CACHE)
2982                 return true;
2983
2984         return false;
2985 }
2986
2987 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
2988 {
2989         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2990
2991         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
2992         if (!mlxsw_sp_rt6)
2993                 return ERR_PTR(-ENOMEM);
2994
2995         /* In case of route replace, replaced route is deleted with
2996          * no notification. Take reference to prevent accessing freed
2997          * memory.
2998          */
2999         mlxsw_sp_rt6->rt = rt;
3000         rt6_hold(rt);
3001
3002         return mlxsw_sp_rt6;
3003 }
3004
3005 #if IS_ENABLED(CONFIG_IPV6)
3006 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3007 {
3008         rt6_release(rt);
3009 }
3010 #else
3011 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3012 {
3013 }
3014 #endif
3015
3016 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3017 {
3018         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
3019         kfree(mlxsw_sp_rt6);
3020 }
3021
3022 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
3023 {
3024         /* RTF_CACHE routes are ignored */
3025         return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
3026 }
3027
3028 static struct rt6_info *
3029 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
3030 {
3031         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3032                                 list)->rt;
3033 }
3034
3035 static struct mlxsw_sp_fib6_entry *
3036 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3037                                  const struct rt6_info *nrt, bool replace)
3038 {
3039         struct mlxsw_sp_fib6_entry *fib6_entry;
3040
3041         if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
3042                 return NULL;
3043
3044         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3045                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3046
3047                 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
3048                  * virtual router.
3049                  */
3050                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3051                         continue;
3052                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3053                         break;
3054                 if (rt->rt6i_metric < nrt->rt6i_metric)
3055                         continue;
3056                 if (rt->rt6i_metric == nrt->rt6i_metric &&
3057                     mlxsw_sp_fib6_rt_can_mp(rt))
3058                         return fib6_entry;
3059                 if (rt->rt6i_metric > nrt->rt6i_metric)
3060                         break;
3061         }
3062
3063         return NULL;
3064 }
3065
3066 static struct mlxsw_sp_rt6 *
3067 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
3068                             const struct rt6_info *rt)
3069 {
3070         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3071
3072         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3073                 if (mlxsw_sp_rt6->rt == rt)
3074                         return mlxsw_sp_rt6;
3075         }
3076
3077         return NULL;
3078 }
3079
3080 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
3081                                   struct mlxsw_sp_nexthop_group *nh_grp,
3082                                   struct mlxsw_sp_nexthop *nh,
3083                                   const struct rt6_info *rt)
3084 {
3085         struct net_device *dev = rt->dst.dev;
3086         struct mlxsw_sp_rif *rif;
3087         int err;
3088
3089         nh->nh_grp = nh_grp;
3090         memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
3091
3092         if (!dev)
3093                 return 0;
3094
3095         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3096         if (!rif)
3097                 return 0;
3098         mlxsw_sp_nexthop_rif_init(nh, rif);
3099
3100         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3101         if (err)
3102                 goto err_nexthop_neigh_init;
3103
3104         return 0;
3105
3106 err_nexthop_neigh_init:
3107         mlxsw_sp_nexthop_rif_fini(nh);
3108         return err;
3109 }
3110
3111 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
3112                                    struct mlxsw_sp_nexthop *nh)
3113 {
3114         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
3115         mlxsw_sp_nexthop_rif_fini(nh);
3116 }
3117
3118 static struct mlxsw_sp_nexthop_group *
3119 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
3120                                struct mlxsw_sp_fib6_entry *fib6_entry)
3121 {
3122         struct mlxsw_sp_nexthop_group *nh_grp;
3123         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3124         struct mlxsw_sp_nexthop *nh;
3125         size_t alloc_size;
3126         int i = 0;
3127         int err;
3128
3129         alloc_size = sizeof(*nh_grp) +
3130                      fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
3131         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
3132         if (!nh_grp)
3133                 return ERR_PTR(-ENOMEM);
3134         INIT_LIST_HEAD(&nh_grp->fib_list);
3135 #if IS_ENABLED(CONFIG_IPV6)
3136         nh_grp->neigh_tbl = &nd_tbl;
3137 #endif
3138         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
3139                                         struct mlxsw_sp_rt6, list);
3140         nh_grp->gateway = !!(mlxsw_sp_rt6->rt->rt6i_flags & RTF_GATEWAY);
3141         nh_grp->count = fib6_entry->nrt6;
3142         for (i = 0; i < nh_grp->count; i++) {
3143                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3144
3145                 nh = &nh_grp->nexthops[i];
3146                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
3147                 if (err)
3148                         goto err_nexthop6_init;
3149                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
3150         }
3151         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3152         return nh_grp;
3153
3154 err_nexthop6_init:
3155         for (i--; i >= 0; i--) {
3156                 nh = &nh_grp->nexthops[i];
3157                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
3158         }
3159         kfree(nh_grp);
3160         return ERR_PTR(err);
3161 }
3162
3163 static void
3164 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
3165                                 struct mlxsw_sp_nexthop_group *nh_grp)
3166 {
3167         struct mlxsw_sp_nexthop *nh;
3168         int i = nh_grp->count;
3169
3170         for (i--; i >= 0; i--) {
3171                 nh = &nh_grp->nexthops[i];
3172                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
3173         }
3174         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3175         WARN_ON(nh_grp->adj_index_valid);
3176         kfree(nh_grp);
3177 }
3178
3179 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
3180                                        struct mlxsw_sp_fib6_entry *fib6_entry)
3181 {
3182         struct mlxsw_sp_nexthop_group *nh_grp;
3183
3184         /* For now, don't consolidate nexthop groups */
3185         nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
3186         if (IS_ERR(nh_grp))
3187                 return PTR_ERR(nh_grp);
3188
3189         list_add_tail(&fib6_entry->common.nexthop_group_node,
3190                       &nh_grp->fib_list);
3191         fib6_entry->common.nh_group = nh_grp;
3192
3193         return 0;
3194 }
3195
3196 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
3197                                         struct mlxsw_sp_fib_entry *fib_entry)
3198 {
3199         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3200
3201         list_del(&fib_entry->nexthop_group_node);
3202         if (!list_empty(&nh_grp->fib_list))
3203                 return;
3204         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
3205 }
3206
3207 static int
3208 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
3209                                struct mlxsw_sp_fib6_entry *fib6_entry)
3210 {
3211         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
3212         int err;
3213
3214         fib6_entry->common.nh_group = NULL;
3215         list_del(&fib6_entry->common.nexthop_group_node);
3216
3217         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
3218         if (err)
3219                 goto err_nexthop6_group_get;
3220
3221         /* In case this entry is offloaded, then the adjacency index
3222          * currently associated with it in the device's table is that
3223          * of the old group. Start using the new one instead.
3224          */
3225         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
3226         if (err)
3227                 goto err_fib_node_entry_add;
3228
3229         if (list_empty(&old_nh_grp->fib_list))
3230                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
3231
3232         return 0;
3233
3234 err_fib_node_entry_add:
3235         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
3236 err_nexthop6_group_get:
3237         list_add_tail(&fib6_entry->common.nexthop_group_node,
3238                       &old_nh_grp->fib_list);
3239         fib6_entry->common.nh_group = old_nh_grp;
3240         return err;
3241 }
3242
3243 static int
3244 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
3245                                 struct mlxsw_sp_fib6_entry *fib6_entry,
3246                                 struct rt6_info *rt)
3247 {
3248         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3249         int err;
3250
3251         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
3252         if (IS_ERR(mlxsw_sp_rt6))
3253                 return PTR_ERR(mlxsw_sp_rt6);
3254
3255         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
3256         fib6_entry->nrt6++;
3257
3258         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
3259         if (err)
3260                 goto err_nexthop6_group_update;
3261
3262         return 0;
3263
3264 err_nexthop6_group_update:
3265         fib6_entry->nrt6--;
3266         list_del(&mlxsw_sp_rt6->list);
3267         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3268         return err;
3269 }
3270
3271 static void
3272 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
3273                                 struct mlxsw_sp_fib6_entry *fib6_entry,
3274                                 struct rt6_info *rt)
3275 {
3276         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3277
3278         mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
3279         if (WARN_ON(!mlxsw_sp_rt6))
3280                 return;
3281
3282         fib6_entry->nrt6--;
3283         list_del(&mlxsw_sp_rt6->list);
3284         mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
3285         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3286 }
3287
3288 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp_fib_entry *fib_entry,
3289                                          const struct rt6_info *rt)
3290 {
3291         /* Packets hitting RTF_REJECT routes need to be discarded by the
3292          * stack. We can rely on their destination device not having a
3293          * RIF (it's the loopback device) and can thus use action type
3294          * local, which will cause them to be trapped with a lower
3295          * priority than packets that need to be locally received.
3296          */
3297         if (rt->rt6i_flags & RTF_LOCAL)
3298                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
3299         else if (rt->rt6i_flags & RTF_REJECT)
3300                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3301         else if (rt->rt6i_flags & RTF_GATEWAY)
3302                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
3303         else
3304                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3305 }
3306
3307 static void
3308 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
3309 {
3310         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
3311
3312         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
3313                                  list) {
3314                 fib6_entry->nrt6--;
3315                 list_del(&mlxsw_sp_rt6->list);
3316                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3317         }
3318 }
3319
3320 static struct mlxsw_sp_fib6_entry *
3321 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
3322                            struct mlxsw_sp_fib_node *fib_node,
3323                            struct rt6_info *rt)
3324 {
3325         struct mlxsw_sp_fib6_entry *fib6_entry;
3326         struct mlxsw_sp_fib_entry *fib_entry;
3327         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3328         int err;
3329
3330         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
3331         if (!fib6_entry)
3332                 return ERR_PTR(-ENOMEM);
3333         fib_entry = &fib6_entry->common;
3334
3335         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
3336         if (IS_ERR(mlxsw_sp_rt6)) {
3337                 err = PTR_ERR(mlxsw_sp_rt6);
3338                 goto err_rt6_create;
3339         }
3340
3341         mlxsw_sp_fib6_entry_type_set(fib_entry, mlxsw_sp_rt6->rt);
3342
3343         INIT_LIST_HEAD(&fib6_entry->rt6_list);
3344         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
3345         fib6_entry->nrt6 = 1;
3346         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
3347         if (err)
3348                 goto err_nexthop6_group_get;
3349
3350         fib_entry->fib_node = fib_node;
3351
3352         return fib6_entry;
3353
3354 err_nexthop6_group_get:
3355         list_del(&mlxsw_sp_rt6->list);
3356         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3357 err_rt6_create:
3358         kfree(fib6_entry);
3359         return ERR_PTR(err);
3360 }
3361
3362 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
3363                                         struct mlxsw_sp_fib6_entry *fib6_entry)
3364 {
3365         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
3366         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
3367         WARN_ON(fib6_entry->nrt6);
3368         kfree(fib6_entry);
3369 }
3370
3371 static struct mlxsw_sp_fib6_entry *
3372 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3373                               const struct rt6_info *nrt, bool replace)
3374 {
3375         struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
3376
3377         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3378                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3379
3380                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3381                         continue;
3382                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3383                         break;
3384                 if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
3385                         if (mlxsw_sp_fib6_rt_can_mp(rt) ==
3386                             mlxsw_sp_fib6_rt_can_mp(nrt))
3387                                 return fib6_entry;
3388                         if (mlxsw_sp_fib6_rt_can_mp(nrt))
3389                                 fallback = fallback ?: fib6_entry;
3390                 }
3391                 if (rt->rt6i_metric > nrt->rt6i_metric)
3392                         return fallback ?: fib6_entry;
3393         }
3394
3395         return fallback;
3396 }
3397
3398 static int
3399 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
3400                                bool replace)
3401 {
3402         struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
3403         struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
3404         struct mlxsw_sp_fib6_entry *fib6_entry;
3405
3406         fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
3407
3408         if (replace && WARN_ON(!fib6_entry))
3409                 return -EINVAL;
3410
3411         if (fib6_entry) {
3412                 list_add_tail(&new6_entry->common.list,
3413                               &fib6_entry->common.list);
3414         } else {
3415                 struct mlxsw_sp_fib6_entry *last;
3416
3417                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3418                         struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
3419
3420                         if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
3421                                 break;
3422                         fib6_entry = last;
3423                 }
3424
3425                 if (fib6_entry)
3426                         list_add(&new6_entry->common.list,
3427                                  &fib6_entry->common.list);
3428                 else
3429                         list_add(&new6_entry->common.list,
3430                                  &fib_node->entry_list);
3431         }
3432
3433         return 0;
3434 }
3435
3436 static void
3437 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
3438 {
3439         list_del(&fib6_entry->common.list);
3440 }
3441
3442 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3443                                          struct mlxsw_sp_fib6_entry *fib6_entry,
3444                                          bool replace)
3445 {
3446         int err;
3447
3448         err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
3449         if (err)
3450                 return err;
3451
3452         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
3453         if (err)
3454                 goto err_fib_node_entry_add;
3455
3456         return 0;
3457
3458 err_fib_node_entry_add:
3459         mlxsw_sp_fib6_node_list_remove(fib6_entry);
3460         return err;
3461 }
3462
3463 static void
3464 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3465                                 struct mlxsw_sp_fib6_entry *fib6_entry)
3466 {
3467         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
3468         mlxsw_sp_fib6_node_list_remove(fib6_entry);
3469 }
3470
3471 static struct mlxsw_sp_fib6_entry *
3472 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
3473                            const struct rt6_info *rt)
3474 {
3475         struct mlxsw_sp_fib6_entry *fib6_entry;
3476         struct mlxsw_sp_fib_node *fib_node;
3477         struct mlxsw_sp_fib *fib;
3478         struct mlxsw_sp_vr *vr;
3479
3480         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
3481         if (!vr)
3482                 return NULL;
3483         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
3484
3485         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
3486                                             sizeof(rt->rt6i_dst.addr),
3487                                             rt->rt6i_dst.plen);
3488         if (!fib_node)
3489                 return NULL;
3490
3491         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3492                 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3493
3494                 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
3495                     rt->rt6i_metric == iter_rt->rt6i_metric &&
3496                     mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
3497                         return fib6_entry;
3498         }
3499
3500         return NULL;
3501 }
3502
3503 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
3504                                         struct mlxsw_sp_fib6_entry *fib6_entry,
3505                                         bool replace)
3506 {
3507         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
3508         struct mlxsw_sp_fib6_entry *replaced;
3509
3510         if (!replace)
3511                 return;
3512
3513         replaced = list_next_entry(fib6_entry, common.list);
3514
3515         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
3516         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
3517         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3518 }
3519
3520 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
3521                                     struct rt6_info *rt, bool replace)
3522 {
3523         struct mlxsw_sp_fib6_entry *fib6_entry;
3524         struct mlxsw_sp_fib_node *fib_node;
3525         int err;
3526
3527         if (mlxsw_sp->router->aborted)
3528                 return 0;
3529
3530         if (rt->rt6i_src.plen)
3531                 return -EINVAL;
3532
3533         if (mlxsw_sp_fib6_rt_should_ignore(rt))
3534                 return 0;
3535
3536         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
3537                                          &rt->rt6i_dst.addr,
3538                                          sizeof(rt->rt6i_dst.addr),
3539                                          rt->rt6i_dst.plen,
3540                                          MLXSW_SP_L3_PROTO_IPV6);
3541         if (IS_ERR(fib_node))
3542                 return PTR_ERR(fib_node);
3543
3544         /* Before creating a new entry, try to append route to an existing
3545          * multipath entry.
3546          */
3547         fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
3548         if (fib6_entry) {
3549                 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
3550                 if (err)
3551                         goto err_fib6_entry_nexthop_add;
3552                 return 0;
3553         }
3554
3555         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
3556         if (IS_ERR(fib6_entry)) {
3557                 err = PTR_ERR(fib6_entry);
3558                 goto err_fib6_entry_create;
3559         }
3560
3561         err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
3562         if (err)
3563                 goto err_fib6_node_entry_link;
3564
3565         mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
3566
3567         return 0;
3568
3569 err_fib6_node_entry_link:
3570         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
3571 err_fib6_entry_create:
3572 err_fib6_entry_nexthop_add:
3573         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3574         return err;
3575 }
3576
3577 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
3578                                      struct rt6_info *rt)
3579 {
3580         struct mlxsw_sp_fib6_entry *fib6_entry;
3581         struct mlxsw_sp_fib_node *fib_node;
3582
3583         if (mlxsw_sp->router->aborted)
3584                 return;
3585
3586         if (mlxsw_sp_fib6_rt_should_ignore(rt))
3587                 return;
3588
3589         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
3590         if (WARN_ON(!fib6_entry))
3591                 return;
3592
3593         /* If route is part of a multipath entry, but not the last one
3594          * removed, then only reduce its nexthop group.
3595          */
3596         if (!list_is_singular(&fib6_entry->rt6_list)) {
3597                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
3598                 return;
3599         }
3600
3601         fib_node = fib6_entry->common.fib_node;
3602
3603         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
3604         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
3605         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3606 }
3607
3608 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
3609                                             enum mlxsw_reg_ralxx_protocol proto,
3610                                             u8 tree_id)
3611 {
3612         char ralta_pl[MLXSW_REG_RALTA_LEN];
3613         char ralst_pl[MLXSW_REG_RALST_LEN];
3614         int i, err;
3615
3616         mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
3617         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
3618         if (err)
3619                 return err;
3620
3621         mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
3622         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
3623         if (err)
3624                 return err;
3625
3626         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
3627                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
3628                 char raltb_pl[MLXSW_REG_RALTB_LEN];
3629                 char ralue_pl[MLXSW_REG_RALUE_LEN];
3630
3631                 if (!mlxsw_sp_vr_is_used(vr))
3632                         continue;
3633
3634                 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
3635                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
3636                                       raltb_pl);
3637                 if (err)
3638                         return err;
3639
3640                 mlxsw_reg_ralue_pack(ralue_pl, proto,
3641                                      MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
3642                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
3643                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
3644                                       ralue_pl);
3645                 if (err)
3646                         return err;
3647         }
3648
3649         return 0;
3650 }
3651
3652 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
3653 {
3654         enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
3655         int err;
3656
3657         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
3658                                                MLXSW_SP_LPM_TREE_MIN);
3659         if (err)
3660                 return err;
3661
3662         proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
3663         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
3664                                                 MLXSW_SP_LPM_TREE_MIN + 1);
3665 }
3666
3667 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
3668                                      struct mlxsw_sp_fib_node *fib_node)
3669 {
3670         struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
3671
3672         list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
3673                                  common.list) {
3674                 bool do_break = &tmp->common.list == &fib_node->entry_list;
3675
3676                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
3677                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3678                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3679                 /* Break when entry list is empty and node was freed.
3680                  * Otherwise, we'll access freed memory in the next
3681                  * iteration.
3682                  */
3683                 if (do_break)
3684                         break;
3685         }
3686 }
3687
3688 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
3689                                      struct mlxsw_sp_fib_node *fib_node)
3690 {
3691         struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
3692
3693         list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
3694                                  common.list) {
3695                 bool do_break = &tmp->common.list == &fib_node->entry_list;
3696
3697                 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
3698                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
3699                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3700                 if (do_break)
3701                         break;
3702         }
3703 }
3704
3705 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
3706                                     struct mlxsw_sp_fib_node *fib_node)
3707 {
3708         switch (fib_node->fib->proto) {
3709         case MLXSW_SP_L3_PROTO_IPV4:
3710                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
3711                 break;
3712         case MLXSW_SP_L3_PROTO_IPV6:
3713                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
3714                 break;
3715         }
3716 }
3717
3718 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
3719                                   struct mlxsw_sp_vr *vr,
3720                                   enum mlxsw_sp_l3proto proto)
3721 {
3722         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
3723         struct mlxsw_sp_fib_node *fib_node, *tmp;
3724
3725         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
3726                 bool do_break = &tmp->list == &fib->node_list;
3727
3728                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
3729                 if (do_break)
3730                         break;
3731         }
3732 }
3733
3734 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
3735 {
3736         int i;
3737
3738         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
3739                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
3740
3741                 if (!mlxsw_sp_vr_is_used(vr))
3742                         continue;
3743                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
3744
3745                 /* If virtual router was only used for IPv4, then it's no
3746                  * longer used.
3747                  */
3748                 if (!mlxsw_sp_vr_is_used(vr))
3749                         continue;
3750                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
3751         }
3752 }
3753
3754 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
3755 {
3756         int err;
3757
3758         if (mlxsw_sp->router->aborted)
3759                 return;
3760         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
3761         mlxsw_sp_router_fib_flush(mlxsw_sp);
3762         mlxsw_sp->router->aborted = true;
3763         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
3764         if (err)
3765                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
3766 }
3767
3768 struct mlxsw_sp_fib_event_work {
3769         struct work_struct work;
3770         union {
3771                 struct fib6_entry_notifier_info fen6_info;
3772                 struct fib_entry_notifier_info fen_info;
3773                 struct fib_rule_notifier_info fr_info;
3774                 struct fib_nh_notifier_info fnh_info;
3775         };
3776         struct mlxsw_sp *mlxsw_sp;
3777         unsigned long event;
3778 };
3779
3780 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
3781 {
3782         struct mlxsw_sp_fib_event_work *fib_work =
3783                 container_of(work, struct mlxsw_sp_fib_event_work, work);
3784         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
3785         struct fib_rule *rule;
3786         bool replace, append;
3787         int err;
3788
3789         /* Protect internal structures from changes */
3790         rtnl_lock();
3791         switch (fib_work->event) {
3792         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
3793         case FIB_EVENT_ENTRY_APPEND: /* fall through */
3794         case FIB_EVENT_ENTRY_ADD:
3795                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
3796                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
3797                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
3798                                                replace, append);
3799                 if (err)
3800                         mlxsw_sp_router_fib_abort(mlxsw_sp);
3801                 fib_info_put(fib_work->fen_info.fi);
3802                 break;
3803         case FIB_EVENT_ENTRY_DEL:
3804                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
3805                 fib_info_put(fib_work->fen_info.fi);
3806                 break;
3807         case FIB_EVENT_RULE_ADD: /* fall through */
3808         case FIB_EVENT_RULE_DEL:
3809                 rule = fib_work->fr_info.rule;
3810                 if (!fib4_rule_default(rule) && !rule->l3mdev)
3811                         mlxsw_sp_router_fib_abort(mlxsw_sp);
3812                 fib_rule_put(rule);
3813                 break;
3814         case FIB_EVENT_NH_ADD: /* fall through */
3815         case FIB_EVENT_NH_DEL:
3816                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
3817                                         fib_work->fnh_info.fib_nh);
3818                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
3819                 break;
3820         }
3821         rtnl_unlock();
3822         kfree(fib_work);
3823 }
3824
3825 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
3826 {
3827         struct mlxsw_sp_fib_event_work *fib_work =
3828                 container_of(work, struct mlxsw_sp_fib_event_work, work);
3829         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
3830         struct fib_rule *rule;
3831         bool replace;
3832         int err;
3833
3834         rtnl_lock();
3835         switch (fib_work->event) {
3836         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
3837         case FIB_EVENT_ENTRY_ADD:
3838                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
3839                 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
3840                                                fib_work->fen6_info.rt, replace);
3841                 if (err)
3842                         mlxsw_sp_router_fib_abort(mlxsw_sp);
3843                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
3844                 break;
3845         case FIB_EVENT_ENTRY_DEL:
3846                 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
3847                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
3848                 break;
3849         case FIB_EVENT_RULE_ADD: /* fall through */
3850         case FIB_EVENT_RULE_DEL:
3851                 rule = fib_work->fr_info.rule;
3852                 if (!fib6_rule_default(rule) && !rule->l3mdev)
3853                         mlxsw_sp_router_fib_abort(mlxsw_sp);
3854                 fib_rule_put(rule);
3855                 break;
3856         }
3857         rtnl_unlock();
3858         kfree(fib_work);
3859 }
3860
3861 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
3862                                        struct fib_notifier_info *info)
3863 {
3864         switch (fib_work->event) {
3865         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
3866         case FIB_EVENT_ENTRY_APPEND: /* fall through */
3867         case FIB_EVENT_ENTRY_ADD: /* fall through */
3868         case FIB_EVENT_ENTRY_DEL:
3869                 memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
3870                 /* Take referece on fib_info to prevent it from being
3871                  * freed while work is queued. Release it afterwards.
3872                  */
3873                 fib_info_hold(fib_work->fen_info.fi);
3874                 break;
3875         case FIB_EVENT_RULE_ADD: /* fall through */
3876         case FIB_EVENT_RULE_DEL:
3877                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
3878                 fib_rule_get(fib_work->fr_info.rule);
3879                 break;
3880         case FIB_EVENT_NH_ADD: /* fall through */
3881         case FIB_EVENT_NH_DEL:
3882                 memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
3883                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
3884                 break;
3885         }
3886 }
3887
3888 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
3889                                        struct fib_notifier_info *info)
3890 {
3891         switch (fib_work->event) {
3892         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
3893         case FIB_EVENT_ENTRY_ADD: /* fall through */
3894         case FIB_EVENT_ENTRY_DEL:
3895                 memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
3896                 rt6_hold(fib_work->fen6_info.rt);
3897                 break;
3898         case FIB_EVENT_RULE_ADD: /* fall through */
3899         case FIB_EVENT_RULE_DEL:
3900                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
3901                 fib_rule_get(fib_work->fr_info.rule);
3902                 break;
3903         }
3904 }
3905
3906 /* Called with rcu_read_lock() */
3907 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
3908                                      unsigned long event, void *ptr)
3909 {
3910         struct mlxsw_sp_fib_event_work *fib_work;
3911         struct fib_notifier_info *info = ptr;
3912         struct mlxsw_sp_router *router;
3913
3914         if (!net_eq(info->net, &init_net))
3915                 return NOTIFY_DONE;
3916
3917         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
3918         if (WARN_ON(!fib_work))
3919                 return NOTIFY_BAD;
3920
3921         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
3922         fib_work->mlxsw_sp = router->mlxsw_sp;
3923         fib_work->event = event;
3924
3925         switch (info->family) {
3926         case AF_INET:
3927                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
3928                 mlxsw_sp_router_fib4_event(fib_work, info);
3929                 break;
3930         case AF_INET6:
3931                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
3932                 mlxsw_sp_router_fib6_event(fib_work, info);
3933                 break;
3934         }
3935
3936         mlxsw_core_schedule_work(&fib_work->work);
3937
3938         return NOTIFY_DONE;
3939 }
3940
3941 static struct mlxsw_sp_rif *
3942 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
3943                          const struct net_device *dev)
3944 {
3945         int i;
3946
3947         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3948                 if (mlxsw_sp->router->rifs[i] &&
3949                     mlxsw_sp->router->rifs[i]->dev == dev)
3950                         return mlxsw_sp->router->rifs[i];
3951
3952         return NULL;
3953 }
3954
3955 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
3956 {
3957         char ritr_pl[MLXSW_REG_RITR_LEN];
3958         int err;
3959
3960         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
3961         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3962         if (WARN_ON_ONCE(err))
3963                 return err;
3964
3965         mlxsw_reg_ritr_enable_set(ritr_pl, false);
3966         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3967 }
3968
3969 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
3970                                           struct mlxsw_sp_rif *rif)
3971 {
3972         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
3973         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
3974         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
3975 }
3976
3977 static bool
3978 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
3979                            unsigned long event)
3980 {
3981         struct inet6_dev *inet6_dev;
3982         bool addr_list_empty = true;
3983         struct in_device *idev;
3984
3985         switch (event) {
3986         case NETDEV_UP:
3987                 return rif == NULL;
3988         case NETDEV_DOWN:
3989                 idev = __in_dev_get_rtnl(dev);
3990                 if (idev && idev->ifa_list)
3991                         addr_list_empty = false;
3992
3993                 inet6_dev = __in6_dev_get(dev);
3994                 if (addr_list_empty && inet6_dev &&
3995                     !list_empty(&inet6_dev->addr_list))
3996                         addr_list_empty = false;
3997
3998                 if (rif && addr_list_empty &&
3999                     !netif_is_l3_slave(rif->dev))
4000                         return true;
4001                 /* It is possible we already removed the RIF ourselves
4002                  * if it was assigned to a netdev that is now a bridge
4003                  * or LAG slave.
4004                  */
4005                 return false;
4006         }
4007
4008         return false;
4009 }
4010
4011 static enum mlxsw_sp_rif_type
4012 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
4013                       const struct net_device *dev)
4014 {
4015         enum mlxsw_sp_fid_type type;
4016
4017         /* RIF type is derived from the type of the underlying FID */
4018         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
4019                 type = MLXSW_SP_FID_TYPE_8021Q;
4020         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
4021                 type = MLXSW_SP_FID_TYPE_8021Q;
4022         else if (netif_is_bridge_master(dev))
4023                 type = MLXSW_SP_FID_TYPE_8021D;
4024         else
4025                 type = MLXSW_SP_FID_TYPE_RFID;
4026
4027         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
4028 }
4029
4030 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
4031 {
4032         int i;
4033
4034         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
4035                 if (!mlxsw_sp->router->rifs[i]) {
4036                         *p_rif_index = i;
4037                         return 0;
4038                 }
4039         }
4040
4041         return -ENOBUFS;
4042 }
4043
4044 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
4045                                                u16 vr_id,
4046                                                struct net_device *l3_dev)
4047 {
4048         struct mlxsw_sp_rif *rif;
4049
4050         rif = kzalloc(rif_size, GFP_KERNEL);
4051         if (!rif)
4052                 return NULL;
4053
4054         INIT_LIST_HEAD(&rif->nexthop_list);
4055         INIT_LIST_HEAD(&rif->neigh_list);
4056         ether_addr_copy(rif->addr, l3_dev->dev_addr);
4057         rif->mtu = l3_dev->mtu;
4058         rif->vr_id = vr_id;
4059         rif->dev = l3_dev;
4060         rif->rif_index = rif_index;
4061
4062         return rif;
4063 }
4064
4065 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
4066                                            u16 rif_index)
4067 {
4068         return mlxsw_sp->router->rifs[rif_index];
4069 }
4070
4071 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
4072 {
4073         return rif->rif_index;
4074 }
4075
4076 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
4077 {
4078         return rif->dev->ifindex;
4079 }
4080
4081 static struct mlxsw_sp_rif *
4082 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
4083                     const struct mlxsw_sp_rif_params *params)
4084 {
4085         u32 tb_id = l3mdev_fib_table(params->dev);
4086         const struct mlxsw_sp_rif_ops *ops;
4087         enum mlxsw_sp_rif_type type;
4088         struct mlxsw_sp_rif *rif;
4089         struct mlxsw_sp_fid *fid;
4090         struct mlxsw_sp_vr *vr;
4091         u16 rif_index;
4092         int err;
4093
4094         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
4095         ops = mlxsw_sp->router->rif_ops_arr[type];
4096
4097         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
4098         if (IS_ERR(vr))
4099                 return ERR_CAST(vr);
4100
4101         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
4102         if (err)
4103                 goto err_rif_index_alloc;
4104
4105         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
4106         if (!rif) {
4107                 err = -ENOMEM;
4108                 goto err_rif_alloc;
4109         }
4110         rif->mlxsw_sp = mlxsw_sp;
4111         rif->ops = ops;
4112
4113         fid = ops->fid_get(rif);
4114         if (IS_ERR(fid)) {
4115                 err = PTR_ERR(fid);
4116                 goto err_fid_get;
4117         }
4118         rif->fid = fid;
4119
4120         if (ops->setup)
4121                 ops->setup(rif, params);
4122
4123         err = ops->configure(rif);
4124         if (err)
4125                 goto err_configure;
4126
4127         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, params->dev->dev_addr,
4128                                   mlxsw_sp_fid_index(fid), true);
4129         if (err)
4130                 goto err_rif_fdb_op;
4131
4132         mlxsw_sp_rif_counters_alloc(rif);
4133         mlxsw_sp_fid_rif_set(fid, rif);
4134         mlxsw_sp->router->rifs[rif_index] = rif;
4135         vr->rif_count++;
4136
4137         return rif;
4138
4139 err_rif_fdb_op:
4140         ops->deconfigure(rif);
4141 err_configure:
4142         mlxsw_sp_fid_put(fid);
4143 err_fid_get:
4144         kfree(rif);
4145 err_rif_alloc:
4146 err_rif_index_alloc:
4147         mlxsw_sp_vr_put(vr);
4148         return ERR_PTR(err);
4149 }
4150
4151 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
4152 {
4153         const struct mlxsw_sp_rif_ops *ops = rif->ops;
4154         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4155         struct mlxsw_sp_fid *fid = rif->fid;
4156         struct mlxsw_sp_vr *vr;
4157
4158         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
4159         vr = &mlxsw_sp->router->vrs[rif->vr_id];
4160
4161         vr->rif_count--;
4162         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
4163         mlxsw_sp_fid_rif_set(fid, NULL);
4164         mlxsw_sp_rif_counters_free(rif);
4165         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->dev->dev_addr,
4166                             mlxsw_sp_fid_index(fid), false);
4167         ops->deconfigure(rif);
4168         mlxsw_sp_fid_put(fid);
4169         kfree(rif);
4170         mlxsw_sp_vr_put(vr);
4171 }
4172
4173 static void
4174 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
4175                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4176 {
4177         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4178
4179         params->vid = mlxsw_sp_port_vlan->vid;
4180         params->lag = mlxsw_sp_port->lagged;
4181         if (params->lag)
4182                 params->lag_id = mlxsw_sp_port->lag_id;
4183         else
4184                 params->system_port = mlxsw_sp_port->local_port;
4185 }
4186
4187 static int
4188 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
4189                                struct net_device *l3_dev)
4190 {
4191         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4192         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
4193         u16 vid = mlxsw_sp_port_vlan->vid;
4194         struct mlxsw_sp_rif *rif;
4195         struct mlxsw_sp_fid *fid;
4196         int err;
4197
4198         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4199         if (!rif) {
4200                 struct mlxsw_sp_rif_params params = {
4201                         .dev = l3_dev,
4202                 };
4203
4204                 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
4205                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
4206                 if (IS_ERR(rif))
4207                         return PTR_ERR(rif);
4208         }
4209
4210         /* FID was already created, just take a reference */
4211         fid = rif->ops->fid_get(rif);
4212         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
4213         if (err)
4214                 goto err_fid_port_vid_map;
4215
4216         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
4217         if (err)
4218                 goto err_port_vid_learning_set;
4219
4220         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
4221                                         BR_STATE_FORWARDING);
4222         if (err)
4223                 goto err_port_vid_stp_set;
4224
4225         mlxsw_sp_port_vlan->fid = fid;
4226
4227         return 0;
4228
4229 err_port_vid_stp_set:
4230         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
4231 err_port_vid_learning_set:
4232         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
4233 err_fid_port_vid_map:
4234         mlxsw_sp_fid_put(fid);
4235         return err;
4236 }
4237
4238 void
4239 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4240 {
4241         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4242         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
4243         u16 vid = mlxsw_sp_port_vlan->vid;
4244
4245         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
4246                 return;
4247
4248         mlxsw_sp_port_vlan->fid = NULL;
4249         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
4250         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
4251         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
4252         /* If router port holds the last reference on the rFID, then the
4253          * associated Sub-port RIF will be destroyed.
4254          */
4255         mlxsw_sp_fid_put(fid);
4256 }
4257
4258 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
4259                                              struct net_device *port_dev,
4260                                              unsigned long event, u16 vid)
4261 {
4262         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
4263         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
4264
4265         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
4266         if (WARN_ON(!mlxsw_sp_port_vlan))
4267                 return -EINVAL;
4268
4269         switch (event) {
4270         case NETDEV_UP:
4271                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
4272                                                       l3_dev);
4273         case NETDEV_DOWN:
4274                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
4275                 break;
4276         }
4277
4278         return 0;
4279 }
4280
4281 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
4282                                         unsigned long event)
4283 {
4284         if (netif_is_bridge_port(port_dev) ||
4285             netif_is_lag_port(port_dev) ||
4286             netif_is_ovs_port(port_dev))
4287                 return 0;
4288
4289         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
4290 }
4291
4292 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
4293                                          struct net_device *lag_dev,
4294                                          unsigned long event, u16 vid)
4295 {
4296         struct net_device *port_dev;
4297         struct list_head *iter;
4298         int err;
4299
4300         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
4301                 if (mlxsw_sp_port_dev_check(port_dev)) {
4302                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
4303                                                                 port_dev,
4304                                                                 event, vid);
4305                         if (err)
4306                                 return err;
4307                 }
4308         }
4309
4310         return 0;
4311 }
4312
4313 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
4314                                        unsigned long event)
4315 {
4316         if (netif_is_bridge_port(lag_dev))
4317                 return 0;
4318
4319         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
4320 }
4321
4322 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
4323                                           unsigned long event)
4324 {
4325         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
4326         struct mlxsw_sp_rif_params params = {
4327                 .dev = l3_dev,
4328         };
4329         struct mlxsw_sp_rif *rif;
4330
4331         switch (event) {
4332         case NETDEV_UP:
4333                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
4334                 if (IS_ERR(rif))
4335                         return PTR_ERR(rif);
4336                 break;
4337         case NETDEV_DOWN:
4338                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4339                 mlxsw_sp_rif_destroy(rif);
4340                 break;
4341         }
4342
4343         return 0;
4344 }
4345
4346 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
4347                                         unsigned long event)
4348 {
4349         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
4350         u16 vid = vlan_dev_vlan_id(vlan_dev);
4351
4352         if (netif_is_bridge_port(vlan_dev))
4353                 return 0;
4354
4355         if (mlxsw_sp_port_dev_check(real_dev))
4356                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
4357                                                          event, vid);
4358         else if (netif_is_lag_master(real_dev))
4359                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
4360                                                      vid);
4361         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
4362                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
4363
4364         return 0;
4365 }
4366
4367 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
4368                                      unsigned long event)
4369 {
4370         if (mlxsw_sp_port_dev_check(dev))
4371                 return mlxsw_sp_inetaddr_port_event(dev, event);
4372         else if (netif_is_lag_master(dev))
4373                 return mlxsw_sp_inetaddr_lag_event(dev, event);
4374         else if (netif_is_bridge_master(dev))
4375                 return mlxsw_sp_inetaddr_bridge_event(dev, event);
4376         else if (is_vlan_dev(dev))
4377                 return mlxsw_sp_inetaddr_vlan_event(dev, event);
4378         else
4379                 return 0;
4380 }
4381
4382 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
4383                             unsigned long event, void *ptr)
4384 {
4385         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
4386         struct net_device *dev = ifa->ifa_dev->dev;
4387         struct mlxsw_sp *mlxsw_sp;
4388         struct mlxsw_sp_rif *rif;
4389         int err = 0;
4390
4391         mlxsw_sp = mlxsw_sp_lower_get(dev);
4392         if (!mlxsw_sp)
4393                 goto out;
4394
4395         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4396         if (!mlxsw_sp_rif_should_config(rif, dev, event))
4397                 goto out;
4398
4399         err = __mlxsw_sp_inetaddr_event(dev, event);
4400 out:
4401         return notifier_from_errno(err);
4402 }
4403
4404 struct mlxsw_sp_inet6addr_event_work {
4405         struct work_struct work;
4406         struct net_device *dev;
4407         unsigned long event;
4408 };
4409
4410 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
4411 {
4412         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
4413                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
4414         struct net_device *dev = inet6addr_work->dev;
4415         unsigned long event = inet6addr_work->event;
4416         struct mlxsw_sp *mlxsw_sp;
4417         struct mlxsw_sp_rif *rif;
4418
4419         rtnl_lock();
4420         mlxsw_sp = mlxsw_sp_lower_get(dev);
4421         if (!mlxsw_sp)
4422                 goto out;
4423
4424         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4425         if (!mlxsw_sp_rif_should_config(rif, dev, event))
4426                 goto out;
4427
4428         __mlxsw_sp_inetaddr_event(dev, event);
4429 out:
4430         rtnl_unlock();
4431         dev_put(dev);
4432         kfree(inet6addr_work);
4433 }
4434
4435 /* Called with rcu_read_lock() */
4436 int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
4437                              unsigned long event, void *ptr)
4438 {
4439         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
4440         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
4441         struct net_device *dev = if6->idev->dev;
4442
4443         if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
4444                 return NOTIFY_DONE;
4445
4446         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
4447         if (!inet6addr_work)
4448                 return NOTIFY_BAD;
4449
4450         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
4451         inet6addr_work->dev = dev;
4452         inet6addr_work->event = event;
4453         dev_hold(dev);
4454         mlxsw_core_schedule_work(&inet6addr_work->work);
4455
4456         return NOTIFY_DONE;
4457 }
4458
4459 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
4460                              const char *mac, int mtu)
4461 {
4462         char ritr_pl[MLXSW_REG_RITR_LEN];
4463         int err;
4464
4465         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
4466         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4467         if (err)
4468                 return err;
4469
4470         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
4471         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
4472         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
4473         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4474 }
4475
4476 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
4477 {
4478         struct mlxsw_sp *mlxsw_sp;
4479         struct mlxsw_sp_rif *rif;
4480         u16 fid_index;
4481         int err;
4482
4483         mlxsw_sp = mlxsw_sp_lower_get(dev);
4484         if (!mlxsw_sp)
4485                 return 0;
4486
4487         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4488         if (!rif)
4489                 return 0;
4490         fid_index = mlxsw_sp_fid_index(rif->fid);
4491
4492         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
4493         if (err)
4494                 return err;
4495
4496         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
4497                                 dev->mtu);
4498         if (err)
4499                 goto err_rif_edit;
4500
4501         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
4502         if (err)
4503                 goto err_rif_fdb_op;
4504
4505         ether_addr_copy(rif->addr, dev->dev_addr);
4506         rif->mtu = dev->mtu;
4507
4508         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
4509
4510         return 0;
4511
4512 err_rif_fdb_op:
4513         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
4514 err_rif_edit:
4515         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
4516         return err;
4517 }
4518
4519 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
4520                                   struct net_device *l3_dev)
4521 {
4522         struct mlxsw_sp_rif *rif;
4523
4524         /* If netdev is already associated with a RIF, then we need to
4525          * destroy it and create a new one with the new virtual router ID.
4526          */
4527         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4528         if (rif)
4529                 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
4530
4531         return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
4532 }
4533
4534 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
4535                                     struct net_device *l3_dev)
4536 {
4537         struct mlxsw_sp_rif *rif;
4538
4539         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4540         if (!rif)
4541                 return;
4542         __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
4543 }
4544
4545 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
4546                                  struct netdev_notifier_changeupper_info *info)
4547 {
4548         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
4549         int err = 0;
4550
4551         if (!mlxsw_sp)
4552                 return 0;
4553
4554         switch (event) {
4555         case NETDEV_PRECHANGEUPPER:
4556                 return 0;
4557         case NETDEV_CHANGEUPPER:
4558                 if (info->linking)
4559                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
4560                 else
4561                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
4562                 break;
4563         }
4564
4565         return err;
4566 }
4567
4568 static struct mlxsw_sp_rif_subport *
4569 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
4570 {
4571         return container_of(rif, struct mlxsw_sp_rif_subport, common);
4572 }
4573
4574 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
4575                                        const struct mlxsw_sp_rif_params *params)
4576 {
4577         struct mlxsw_sp_rif_subport *rif_subport;
4578
4579         rif_subport = mlxsw_sp_rif_subport_rif(rif);
4580         rif_subport->vid = params->vid;
4581         rif_subport->lag = params->lag;
4582         if (params->lag)
4583                 rif_subport->lag_id = params->lag_id;
4584         else
4585                 rif_subport->system_port = params->system_port;
4586 }
4587
4588 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
4589 {
4590         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4591         struct mlxsw_sp_rif_subport *rif_subport;
4592         char ritr_pl[MLXSW_REG_RITR_LEN];
4593
4594         rif_subport = mlxsw_sp_rif_subport_rif(rif);
4595         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
4596                             rif->rif_index, rif->vr_id, rif->dev->mtu,
4597                             rif->dev->dev_addr);
4598         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
4599                                   rif_subport->lag ? rif_subport->lag_id :
4600                                                      rif_subport->system_port,
4601                                   rif_subport->vid);
4602
4603         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4604 }
4605
4606 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
4607 {
4608         return mlxsw_sp_rif_subport_op(rif, true);
4609 }
4610
4611 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
4612 {
4613         mlxsw_sp_rif_subport_op(rif, false);
4614 }
4615
4616 static struct mlxsw_sp_fid *
4617 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
4618 {
4619         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
4620 }
4621
4622 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
4623         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
4624         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
4625         .setup                  = mlxsw_sp_rif_subport_setup,
4626         .configure              = mlxsw_sp_rif_subport_configure,
4627         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
4628         .fid_get                = mlxsw_sp_rif_subport_fid_get,
4629 };
4630
4631 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
4632                                     enum mlxsw_reg_ritr_if_type type,
4633                                     u16 vid_fid, bool enable)
4634 {
4635         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4636         char ritr_pl[MLXSW_REG_RITR_LEN];
4637
4638         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
4639                             rif->dev->mtu, rif->dev->dev_addr);
4640         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
4641
4642         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4643 }
4644
4645 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
4646 {
4647         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
4648 }
4649
4650 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
4651 {
4652         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4653         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
4654         int err;
4655
4656         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
4657         if (err)
4658                 return err;
4659
4660         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
4661                                      mlxsw_sp_router_port(mlxsw_sp), true);
4662         if (err)
4663                 goto err_fid_mc_flood_set;
4664
4665         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
4666                                      mlxsw_sp_router_port(mlxsw_sp), true);
4667         if (err)
4668                 goto err_fid_bc_flood_set;
4669
4670         return 0;
4671
4672 err_fid_bc_flood_set:
4673         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
4674                                mlxsw_sp_router_port(mlxsw_sp), false);
4675 err_fid_mc_flood_set:
4676         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
4677         return err;
4678 }
4679
4680 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
4681 {
4682         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4683         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
4684
4685         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
4686                                mlxsw_sp_router_port(mlxsw_sp), false);
4687         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
4688                                mlxsw_sp_router_port(mlxsw_sp), false);
4689         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
4690 }
4691
4692 static struct mlxsw_sp_fid *
4693 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
4694 {
4695         u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
4696
4697         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
4698 }
4699
4700 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
4701         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
4702         .rif_size               = sizeof(struct mlxsw_sp_rif),
4703         .configure              = mlxsw_sp_rif_vlan_configure,
4704         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
4705         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
4706 };
4707
4708 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
4709 {
4710         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4711         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
4712         int err;
4713
4714         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
4715                                        true);
4716         if (err)
4717                 return err;
4718
4719         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
4720                                      mlxsw_sp_router_port(mlxsw_sp), true);
4721         if (err)
4722                 goto err_fid_mc_flood_set;
4723
4724         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
4725                                      mlxsw_sp_router_port(mlxsw_sp), true);
4726         if (err)
4727                 goto err_fid_bc_flood_set;
4728
4729         return 0;
4730
4731 err_fid_bc_flood_set:
4732         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
4733                                mlxsw_sp_router_port(mlxsw_sp), false);
4734 err_fid_mc_flood_set:
4735         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
4736         return err;
4737 }
4738
4739 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
4740 {
4741         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4742         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
4743
4744         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
4745                                mlxsw_sp_router_port(mlxsw_sp), false);
4746         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
4747                                mlxsw_sp_router_port(mlxsw_sp), false);
4748         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
4749 }
4750
4751 static struct mlxsw_sp_fid *
4752 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
4753 {
4754         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
4755 }
4756
4757 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
4758         .type                   = MLXSW_SP_RIF_TYPE_FID,
4759         .rif_size               = sizeof(struct mlxsw_sp_rif),
4760         .configure              = mlxsw_sp_rif_fid_configure,
4761         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
4762         .fid_get                = mlxsw_sp_rif_fid_fid_get,
4763 };
4764
4765 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
4766         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
4767         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_ops,
4768         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
4769 };
4770
4771 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
4772 {
4773         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
4774
4775         mlxsw_sp->router->rifs = kcalloc(max_rifs,
4776                                          sizeof(struct mlxsw_sp_rif *),
4777                                          GFP_KERNEL);
4778         if (!mlxsw_sp->router->rifs)
4779                 return -ENOMEM;
4780
4781         mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
4782
4783         return 0;
4784 }
4785
4786 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
4787 {
4788         int i;
4789
4790         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
4791                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
4792
4793         kfree(mlxsw_sp->router->rifs);
4794 }
4795
4796 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
4797 {
4798         struct mlxsw_sp_router *router;
4799
4800         /* Flush pending FIB notifications and then flush the device's
4801          * table before requesting another dump. The FIB notification
4802          * block is unregistered, so no need to take RTNL.
4803          */
4804         mlxsw_core_flush_owq();
4805         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
4806         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
4807 }
4808
4809 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
4810 {
4811         char rgcr_pl[MLXSW_REG_RGCR_LEN];
4812         u64 max_rifs;
4813         int err;
4814
4815         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
4816                 return -EIO;
4817         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
4818
4819         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
4820         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
4821         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
4822         if (err)
4823                 return err;
4824         return 0;
4825 }
4826
4827 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
4828 {
4829         char rgcr_pl[MLXSW_REG_RGCR_LEN];
4830
4831         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
4832         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
4833 }
4834
4835 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
4836 {
4837         struct mlxsw_sp_router *router;
4838         int err;
4839
4840         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
4841         if (!router)
4842                 return -ENOMEM;
4843         mlxsw_sp->router = router;
4844         router->mlxsw_sp = mlxsw_sp;
4845
4846         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
4847         err = __mlxsw_sp_router_init(mlxsw_sp);
4848         if (err)
4849                 goto err_router_init;
4850
4851         err = mlxsw_sp_rifs_init(mlxsw_sp);
4852         if (err)
4853                 goto err_rifs_init;
4854
4855         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
4856                               &mlxsw_sp_nexthop_ht_params);
4857         if (err)
4858                 goto err_nexthop_ht_init;
4859
4860         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
4861                               &mlxsw_sp_nexthop_group_ht_params);
4862         if (err)
4863                 goto err_nexthop_group_ht_init;
4864
4865         err = mlxsw_sp_lpm_init(mlxsw_sp);
4866         if (err)
4867                 goto err_lpm_init;
4868
4869         err = mlxsw_sp_vrs_init(mlxsw_sp);
4870         if (err)
4871                 goto err_vrs_init;
4872
4873         err = mlxsw_sp_neigh_init(mlxsw_sp);
4874         if (err)
4875                 goto err_neigh_init;
4876
4877         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
4878         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
4879                                     mlxsw_sp_router_fib_dump_flush);
4880         if (err)
4881                 goto err_register_fib_notifier;
4882
4883         return 0;
4884
4885 err_register_fib_notifier:
4886         mlxsw_sp_neigh_fini(mlxsw_sp);
4887 err_neigh_init:
4888         mlxsw_sp_vrs_fini(mlxsw_sp);
4889 err_vrs_init:
4890         mlxsw_sp_lpm_fini(mlxsw_sp);
4891 err_lpm_init:
4892         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
4893 err_nexthop_group_ht_init:
4894         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
4895 err_nexthop_ht_init:
4896         mlxsw_sp_rifs_fini(mlxsw_sp);
4897 err_rifs_init:
4898         __mlxsw_sp_router_fini(mlxsw_sp);
4899 err_router_init:
4900         kfree(mlxsw_sp->router);
4901         return err;
4902 }
4903
4904 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
4905 {
4906         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
4907         mlxsw_sp_neigh_fini(mlxsw_sp);
4908         mlxsw_sp_vrs_fini(mlxsw_sp);
4909         mlxsw_sp_lpm_fini(mlxsw_sp);
4910         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
4911         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
4912         mlxsw_sp_rifs_fini(mlxsw_sp);
4913         __mlxsw_sp_router_fini(mlxsw_sp);
4914         kfree(mlxsw_sp->router);
4915 }