drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  * 3. Neither the names of the copyright holders nor the names of its
  18  *    contributors may be used to endorse or promote products derived from
  19  *    this software without specific prior written permission.
  20  *
  21  * Alternatively, this software may be distributed under the terms of the
  22  * GNU General Public License ("GPL") version 2 as published by the Free
  23  * Software Foundation.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  26  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  29  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  35  * POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 #include <linux/kernel.h>
  39 #include <linux/types.h>
  40 #include <linux/rhashtable.h>
  41 #include <linux/bitops.h>
  42 #include <linux/in6.h>
  43 #include <linux/notifier.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/netdevice.h>
  46 #include <linux/if_bridge.h>
  47 #include <linux/socket.h>
  48 #include <linux/route.h>
  49 #include <net/netevent.h>
  50 #include <net/neighbour.h>
  51 #include <net/arp.h>
  52 #include <net/ip_fib.h>
  53 #include <net/ip6_fib.h>
  54 #include <net/fib_rules.h>
  55 #include <net/ip_tunnels.h>
  56 #include <net/l3mdev.h>
  57 #include <net/addrconf.h>
  58 #include <net/ndisc.h>
  59 #include <net/ipv6.h>
  60 #include <net/fib_notifier.h>
  61
  62 #include "spectrum.h"
  63 #include "core.h"
  64 #include "reg.h"
  65 #include "spectrum_cnt.h"
  66 #include "spectrum_dpipe.h"
  67 #include "spectrum_ipip.h"
  68 #include "spectrum_router.h"
  69
  70 struct mlxsw_sp_vr;
  71 struct mlxsw_sp_lpm_tree;
  72 struct mlxsw_sp_rif_ops;
  73
  74 struct mlxsw_sp_router {
  75         struct mlxsw_sp *mlxsw_sp;
  76         struct mlxsw_sp_rif **rifs;
  77         struct mlxsw_sp_vr *vrs;
  78         struct rhashtable neigh_ht;
  79         struct rhashtable nexthop_group_ht;
  80         struct rhashtable nexthop_ht;
  81         struct {
  82                 struct mlxsw_sp_lpm_tree *trees;
  83                 unsigned int tree_count;
  84         } lpm;
  85         struct {
  86                 struct delayed_work dw;
  87                 unsigned long interval; /* ms */
  88         } neighs_update;
  89         struct delayed_work nexthop_probe_dw;
  90 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  91         struct list_head nexthop_neighs_list;
  92         struct list_head ipip_list;
  93         bool aborted;
  94         struct notifier_block fib_nb;
  95         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  96         const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
  97 };
  98
  99 struct mlxsw_sp_rif {
 100         struct list_head nexthop_list;
 101         struct list_head neigh_list;
 102         struct net_device *dev;
 103         struct mlxsw_sp_fid *fid;
 104         unsigned char addr[ETH_ALEN];
 105         int mtu;
 106         u16 rif_index;
 107         u16 vr_id;
 108         const struct mlxsw_sp_rif_ops *ops;
 109         struct mlxsw_sp *mlxsw_sp;
 110
 111         unsigned int counter_ingress;
 112         bool counter_ingress_valid;
 113         unsigned int counter_egress;
 114         bool counter_egress_valid;
 115 };
 116
 117 struct mlxsw_sp_rif_params {
 118         struct net_device *dev;
 119         union {
 120                 u16 system_port;
 121                 u16 lag_id;
 122         };
 123         u16 vid;
 124         bool lag;
 125 };
 126
 127 struct mlxsw_sp_rif_subport {
 128         struct mlxsw_sp_rif common;
 129         union {
 130                 u16 system_port;
 131                 u16 lag_id;
 132         };
 133         u16 vid;
 134         bool lag;
 135 };
 136
 137 struct mlxsw_sp_rif_ipip_lb {
 138         struct mlxsw_sp_rif common;
 139         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 140         u16 ul_vr_id; /* Reserved for Spectrum-2. */
 141 };
 142
 143 struct mlxsw_sp_rif_params_ipip_lb {
 144         struct mlxsw_sp_rif_params common;
 145         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 146 };
 147
 148 struct mlxsw_sp_rif_ops {
 149         enum mlxsw_sp_rif_type type;
 150         size_t rif_size;
 151
 152         void (*setup)(struct mlxsw_sp_rif *rif,
 153                       const struct mlxsw_sp_rif_params *params);
 154         int (*configure)(struct mlxsw_sp_rif *rif);
 155         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 156         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 157 };
 158
 159 static unsigned int *
 160 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 161                            enum mlxsw_sp_rif_counter_dir dir)
 162 {
 163         switch (dir) {
 164         case MLXSW_SP_RIF_COUNTER_EGRESS:
 165                 return &rif->counter_egress;
 166         case MLXSW_SP_RIF_COUNTER_INGRESS:
 167                 return &rif->counter_ingress;
 168         }
 169         return NULL;
 170 }
 171
 172 static bool
 173 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 174                                enum mlxsw_sp_rif_counter_dir dir)
 175 {
 176         switch (dir) {
 177         case MLXSW_SP_RIF_COUNTER_EGRESS:
 178                 return rif->counter_egress_valid;
 179         case MLXSW_SP_RIF_COUNTER_INGRESS:
 180                 return rif->counter_ingress_valid;
 181         }
 182         return false;
 183 }
 184
 185 static void
 186 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 187                                enum mlxsw_sp_rif_counter_dir dir,
 188                                bool valid)
 189 {
 190         switch (dir) {
 191         case MLXSW_SP_RIF_COUNTER_EGRESS:
 192                 rif->counter_egress_valid = valid;
 193                 break;
 194         case MLXSW_SP_RIF_COUNTER_INGRESS:
 195                 rif->counter_ingress_valid = valid;
 196                 break;
 197         }
 198 }
 199
 200 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 201                                      unsigned int counter_index, bool enable,
 202                                      enum mlxsw_sp_rif_counter_dir dir)
 203 {
 204         char ritr_pl[MLXSW_REG_RITR_LEN];
 205         bool is_egress = false;
 206         int err;
 207
 208         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 209                 is_egress = true;
 210         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 211         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 212         if (err)
 213                 return err;
 214
 215         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 216                                     is_egress);
 217         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 218 }
 219
 220 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 221                                    struct mlxsw_sp_rif *rif,
 222                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 223 {
 224         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 225         unsigned int *p_counter_index;
 226         bool valid;
 227         int err;
 228
 229         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 230         if (!valid)
 231                 return -EINVAL;
 232
 233         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 234         if (!p_counter_index)
 235                 return -EINVAL;
 236         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 237                              MLXSW_REG_RICNT_OPCODE_NOP);
 238         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 239         if (err)
 240                 return err;
 241         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 242         return 0;
 243 }
 244
 245 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 246                                       unsigned int counter_index)
 247 {
 248         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 249
 250         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 251                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 252         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 253 }
 254
 255 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 256                                struct mlxsw_sp_rif *rif,
 257                                enum mlxsw_sp_rif_counter_dir dir)
 258 {
 259         unsigned int *p_counter_index;
 260         int err;
 261
 262         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 263         if (!p_counter_index)
 264                 return -EINVAL;
 265         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 266                                      p_counter_index);
 267         if (err)
 268                 return err;
 269
 270         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 271         if (err)
 272                 goto err_counter_clear;
 273
 274         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 275                                         *p_counter_index, true, dir);
 276         if (err)
 277                 goto err_counter_edit;
 278         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 279         return 0;
 280
 281 err_counter_edit:
 282 err_counter_clear:
 283         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 284                               *p_counter_index);
 285         return err;
 286 }
 287
 288 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 289                                struct mlxsw_sp_rif *rif,
 290                                enum mlxsw_sp_rif_counter_dir dir)
 291 {
 292         unsigned int *p_counter_index;
 293
 294         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 295                 return;
 296
 297         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 298         if (WARN_ON(!p_counter_index))
 299                 return;
 300         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 301                                   *p_counter_index, false, dir);
 302         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 303                               *p_counter_index);
 304         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 305 }
 306
 307 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 308 {
 309         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 310         struct devlink *devlink;
 311
 312         devlink = priv_to_devlink(mlxsw_sp->core);
 313         if (!devlink_dpipe_table_counter_enabled(devlink,
 314                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 315                 return;
 316         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 317 }
 318
 319 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 320 {
 321         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 322
 323         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 324 }
 325
 326 static struct mlxsw_sp_rif *
 327 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 328                          const struct net_device *dev);
 329
 330 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 331
 332 struct mlxsw_sp_prefix_usage {
 333         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 334 };
 335
 336 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 337         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 338
 339 static bool
 340 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 341                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 342 {
 343         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 344 }
 345
 346 static bool
 347 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
 348 {
 349         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
 350
 351         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 352 }
 353
 354 static void
 355 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 356                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 357 {
 358         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 359 }
 360
 361 static void
 362 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 363                           unsigned char prefix_len)
 364 {
 365         set_bit(prefix_len, prefix_usage->b);
 366 }
 367
 368 static void
 369 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 370                             unsigned char prefix_len)
 371 {
 372         clear_bit(prefix_len, prefix_usage->b);
 373 }
 374
 375 struct mlxsw_sp_fib_key {
 376         unsigned char addr[sizeof(struct in6_addr)];
 377         unsigned char prefix_len;
 378 };
 379
 380 enum mlxsw_sp_fib_entry_type {
 381         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 382         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 383         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 384
 385         /* This is a special case of local delivery, where a packet should be
 386          * decapsulated on reception. Note that there is no corresponding ENCAP,
 387          * because that's a type of next hop, not of FIB entry. (There can be
 388          * several next hops in a REMOTE entry, and some of them may be
 389          * encapsulating entries.)
 390          */
 391         MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 392 };
 393
 394 struct mlxsw_sp_nexthop_group;
 395 struct mlxsw_sp_fib;
 396
 397 struct mlxsw_sp_fib_node {
 398         struct list_head entry_list;
 399         struct list_head list;
 400         struct rhash_head ht_node;
 401         struct mlxsw_sp_fib *fib;
 402         struct mlxsw_sp_fib_key key;
 403 };
 404
 405 struct mlxsw_sp_fib_entry_decap {
 406         struct mlxsw_sp_ipip_entry *ipip_entry;
 407         u32 tunnel_index;
 408 };
 409
 410 struct mlxsw_sp_fib_entry {
 411         struct list_head list;
 412         struct mlxsw_sp_fib_node *fib_node;
 413         enum mlxsw_sp_fib_entry_type type;
 414         struct list_head nexthop_group_node;
 415         struct mlxsw_sp_nexthop_group *nh_group;
 416         struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 417 };
 418
 419 struct mlxsw_sp_fib4_entry {
 420         struct mlxsw_sp_fib_entry common;
 421         u32 tb_id;
 422         u32 prio;
 423         u8 tos;
 424         u8 type;
 425 };
 426
 427 struct mlxsw_sp_fib6_entry {
 428         struct mlxsw_sp_fib_entry common;
 429         struct list_head rt6_list;
 430         unsigned int nrt6;
 431 };
 432
 433 struct mlxsw_sp_rt6 {
 434         struct list_head list;
 435         struct rt6_info *rt;
 436 };
 437
 438 struct mlxsw_sp_lpm_tree {
 439         u8 id; /* tree ID */
 440         unsigned int ref_count;
 441         enum mlxsw_sp_l3proto proto;
 442         struct mlxsw_sp_prefix_usage prefix_usage;
 443 };
 444
 445 struct mlxsw_sp_fib {
 446         struct rhashtable ht;
 447         struct list_head node_list;
 448         struct mlxsw_sp_vr *vr;
 449         struct mlxsw_sp_lpm_tree *lpm_tree;
 450         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 451         struct mlxsw_sp_prefix_usage prefix_usage;
 452         enum mlxsw_sp_l3proto proto;
 453 };
 454
 455 struct mlxsw_sp_vr {
 456         u16 id; /* virtual router ID */
 457         u32 tb_id; /* kernel fib table id */
 458         unsigned int rif_count;
 459         struct mlxsw_sp_fib *fib4;
 460         struct mlxsw_sp_fib *fib6;
 461 };
 462
 463 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 464
 465 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 466                                                 enum mlxsw_sp_l3proto proto)
 467 {
 468         struct mlxsw_sp_fib *fib;
 469         int err;
 470
 471         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 472         if (!fib)
 473                 return ERR_PTR(-ENOMEM);
 474         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 475         if (err)
 476                 goto err_rhashtable_init;
 477         INIT_LIST_HEAD(&fib->node_list);
 478         fib->proto = proto;
 479         fib->vr = vr;
 480         return fib;
 481
 482 err_rhashtable_init:
 483         kfree(fib);
 484         return ERR_PTR(err);
 485 }
 486
 487 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 488 {
 489         WARN_ON(!list_empty(&fib->node_list));
 490         WARN_ON(fib->lpm_tree);
 491         rhashtable_destroy(&fib->ht);
 492         kfree(fib);
 493 }
 494
 495 static struct mlxsw_sp_lpm_tree *
 496 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 497 {
 498         static struct mlxsw_sp_lpm_tree *lpm_tree;
 499         int i;
 500
 501         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 502                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 503                 if (lpm_tree->ref_count == 0)
 504                         return lpm_tree;
 505         }
 506         return NULL;
 507 }
 508
 509 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 510                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 511 {
 512         char ralta_pl[MLXSW_REG_RALTA_LEN];
 513
 514         mlxsw_reg_ralta_pack(ralta_pl, true,
 515                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 516                              lpm_tree->id);
 517         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 518 }
 519
 520 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 521                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 522 {
 523         char ralta_pl[MLXSW_REG_RALTA_LEN];
 524
 525         mlxsw_reg_ralta_pack(ralta_pl, false,
 526                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 527                              lpm_tree->id);
 528         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 529 }
 530
 531 static int
 532 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 533                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 534                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 535 {
 536         char ralst_pl[MLXSW_REG_RALST_LEN];
 537         u8 root_bin = 0;
 538         u8 prefix;
 539         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 540
 541         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 542                 root_bin = prefix;
 543
 544         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 545         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 546                 if (prefix == 0)
 547                         continue;
 548                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 549                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 550                 last_prefix = prefix;
 551         }
 552         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 553 }
 554
 555 static struct mlxsw_sp_lpm_tree *
 556 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 557                          struct mlxsw_sp_prefix_usage *prefix_usage,
 558                          enum mlxsw_sp_l3proto proto)
 559 {
 560         struct mlxsw_sp_lpm_tree *lpm_tree;
 561         int err;
 562
 563         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 564         if (!lpm_tree)
 565                 return ERR_PTR(-EBUSY);
 566         lpm_tree->proto = proto;
 567         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 568         if (err)
 569                 return ERR_PTR(err);
 570
 571         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 572                                                 lpm_tree);
 573         if (err)
 574                 goto err_left_struct_set;
 575         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 576                sizeof(lpm_tree->prefix_usage));
 577         return lpm_tree;
 578
 579 err_left_struct_set:
 580         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 581         return ERR_PTR(err);
 582 }
 583
 584 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 585                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 586 {
 587         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 588 }
 589
 590 static struct mlxsw_sp_lpm_tree *
 591 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 592                       struct mlxsw_sp_prefix_usage *prefix_usage,
 593                       enum mlxsw_sp_l3proto proto)
 594 {
 595         struct mlxsw_sp_lpm_tree *lpm_tree;
 596         int i;
 597
 598         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 599                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 600                 if (lpm_tree->ref_count != 0 &&
 601                     lpm_tree->proto == proto &&
 602                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 603                                              prefix_usage))
 604                         return lpm_tree;
 605         }
 606         return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
 607 }
 608
 609 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 610 {
 611         lpm_tree->ref_count++;
 612 }
 613
 614 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 615                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 616 {
 617         if (--lpm_tree->ref_count == 0)
 618                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 619 }
 620
 621 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 622
 623 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 624 {
 625         struct mlxsw_sp_lpm_tree *lpm_tree;
 626         u64 max_trees;
 627         int i;
 628
 629         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 630                 return -EIO;
 631
 632         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 633         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 634         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 635                                              sizeof(struct mlxsw_sp_lpm_tree),
 636                                              GFP_KERNEL);
 637         if (!mlxsw_sp->router->lpm.trees)
 638                 return -ENOMEM;
 639
 640         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 641                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 642                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 643         }
 644
 645         return 0;
 646 }
 647
 648 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 649 {
 650         kfree(mlxsw_sp->router->lpm.trees);
 651 }
 652
 653 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 654 {
 655         return !!vr->fib4 || !!vr->fib6;
 656 }
 657
 658 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 659 {
 660         struct mlxsw_sp_vr *vr;
 661         int i;
 662
 663         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 664                 vr = &mlxsw_sp->router->vrs[i];
 665                 if (!mlxsw_sp_vr_is_used(vr))
 666                         return vr;
 667         }
 668         return NULL;
 669 }
 670
 671 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 672                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 673 {
 674         char raltb_pl[MLXSW_REG_RALTB_LEN];
 675
 676         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 677                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 678                              tree_id);
 679         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 680 }
 681
 682 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 683                                        const struct mlxsw_sp_fib *fib)
 684 {
 685         char raltb_pl[MLXSW_REG_RALTB_LEN];
 686
 687         /* Bind to tree 0 which is default */
 688         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 689                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 690         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 691 }
 692
 693 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 694 {
 695         /* For our purpose, squash main and local table into one */
 696         if (tb_id == RT_TABLE_LOCAL)
 697                 tb_id = RT_TABLE_MAIN;
 698         return tb_id;
 699 }
 700
 701 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 702                                             u32 tb_id)
 703 {
 704         struct mlxsw_sp_vr *vr;
 705         int i;
 706
 707         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 708
 709         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 710                 vr = &mlxsw_sp->router->vrs[i];
 711                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 712                         return vr;
 713         }
 714         return NULL;
 715 }
 716
 717 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 718                                             enum mlxsw_sp_l3proto proto)
 719 {
 720         switch (proto) {
 721         case MLXSW_SP_L3_PROTO_IPV4:
 722                 return vr->fib4;
 723         case MLXSW_SP_L3_PROTO_IPV6:
 724                 return vr->fib6;
 725         }
 726         return NULL;
 727 }
 728
 729 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 730                                               u32 tb_id)
 731 {
 732         struct mlxsw_sp_vr *vr;
 733         int err;
 734
 735         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 736         if (!vr)
 737                 return ERR_PTR(-EBUSY);
 738         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 739         if (IS_ERR(vr->fib4))
 740                 return ERR_CAST(vr->fib4);
 741         vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
 742         if (IS_ERR(vr->fib6)) {
 743                 err = PTR_ERR(vr->fib6);
 744                 goto err_fib6_create;
 745         }
 746         vr->tb_id = tb_id;
 747         return vr;
 748
 749 err_fib6_create:
 750         mlxsw_sp_fib_destroy(vr->fib4);
 751         vr->fib4 = NULL;
 752         return ERR_PTR(err);
 753 }
 754
 755 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 756 {
 757         mlxsw_sp_fib_destroy(vr->fib6);
 758         vr->fib6 = NULL;
 759         mlxsw_sp_fib_destroy(vr->fib4);
 760         vr->fib4 = NULL;
 761 }
 762
 763 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 764 {
 765         struct mlxsw_sp_vr *vr;
 766
 767         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 768         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 769         if (!vr)
 770                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 771         return vr;
 772 }
 773
 774 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 775 {
 776         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 777             list_empty(&vr->fib6->node_list))
 778                 mlxsw_sp_vr_destroy(vr);
 779 }
 780
 781 static bool
 782 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 783                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 784 {
 785         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 786
 787         if (!mlxsw_sp_vr_is_used(vr))
 788                 return false;
 789         if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
 790                 return true;
 791         return false;
 792 }
 793
 794 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 795                                         struct mlxsw_sp_fib *fib,
 796                                         struct mlxsw_sp_lpm_tree *new_tree)
 797 {
 798         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 799         int err;
 800
 801         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 802         if (err)
 803                 return err;
 804         fib->lpm_tree = new_tree;
 805         mlxsw_sp_lpm_tree_hold(new_tree);
 806         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 807         return 0;
 808 }
 809
 810 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 811                                          struct mlxsw_sp_fib *fib,
 812                                          struct mlxsw_sp_lpm_tree *new_tree)
 813 {
 814         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 815         enum mlxsw_sp_l3proto proto = fib->proto;
 816         u8 old_id, new_id = new_tree->id;
 817         struct mlxsw_sp_vr *vr;
 818         int i, err;
 819
 820         if (!old_tree)
 821                 goto no_replace;
 822         old_id = old_tree->id;
 823
 824         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 825                 vr = &mlxsw_sp->router->vrs[i];
 826                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 827                         continue;
 828                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 829                                                    mlxsw_sp_vr_fib(vr, proto),
 830                                                    new_tree);
 831                 if (err)
 832                         goto err_tree_replace;
 833         }
 834
 835         return 0;
 836
 837 err_tree_replace:
 838         for (i--; i >= 0; i--) {
 839                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 840                         continue;
 841                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 842                                              mlxsw_sp_vr_fib(vr, proto),
 843                                              old_tree);
 844         }
 845         return err;
 846
 847 no_replace:
 848         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 849         if (err)
 850                 return err;
 851         fib->lpm_tree = new_tree;
 852         mlxsw_sp_lpm_tree_hold(new_tree);
 853         return 0;
 854 }
 855
 856 static void
 857 mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
 858                       enum mlxsw_sp_l3proto proto,
 859                       struct mlxsw_sp_prefix_usage *req_prefix_usage)
 860 {
 861         int i;
 862
 863         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 864                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
 865                 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 866                 unsigned char prefix;
 867
 868                 if (!mlxsw_sp_vr_is_used(vr))
 869                         continue;
 870                 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
 871                         mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
 872         }
 873 }
 874
 875 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 876 {
 877         struct mlxsw_sp_vr *vr;
 878         u64 max_vrs;
 879         int i;
 880
 881         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 882                 return -EIO;
 883
 884         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 885         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 886                                         GFP_KERNEL);
 887         if (!mlxsw_sp->router->vrs)
 888                 return -ENOMEM;
 889
 890         for (i = 0; i < max_vrs; i++) {
 891                 vr = &mlxsw_sp->router->vrs[i];
 892                 vr->id = i;
 893         }
 894
 895         return 0;
 896 }
 897
 898 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 899
 900 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 901 {
 902         /* At this stage we're guaranteed not to have new incoming
 903          * FIB notifications and the work queue is free from FIBs
 904          * sitting on top of mlxsw netdevs. However, we can still
 905          * have other FIBs queued. Flush the queue before flushing
 906          * the device's tables. No need for locks, as we're the only
 907          * writer.
 908          */
 909         mlxsw_core_flush_owq();
 910         mlxsw_sp_router_fib_flush(mlxsw_sp);
 911         kfree(mlxsw_sp->router->vrs);
 912 }
 913
 914 static struct net_device *
 915 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
 916 {
 917         struct ip_tunnel *tun = netdev_priv(ol_dev);
 918         struct net *net = dev_net(ol_dev);
 919
 920         return __dev_get_by_index(net, tun->parms.link);
 921 }
 922
 923 static u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
 924 {
 925         struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
 926
 927         if (d)
 928                 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
 929         else
 930                 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
 931 }
 932
 933 static struct mlxsw_sp_rif *
 934 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 935                     const struct mlxsw_sp_rif_params *params);
 936
 937 static struct mlxsw_sp_rif_ipip_lb *
 938 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
 939                                 enum mlxsw_sp_ipip_type ipipt,
 940                                 struct net_device *ol_dev)
 941 {
 942         struct mlxsw_sp_rif_params_ipip_lb lb_params;
 943         const struct mlxsw_sp_ipip_ops *ipip_ops;
 944         struct mlxsw_sp_rif *rif;
 945
 946         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
 947         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
 948                 .common.dev = ol_dev,
 949                 .common.lag = false,
 950                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
 951         };
 952
 953         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common);
 954         if (IS_ERR(rif))
 955                 return ERR_CAST(rif);
 956         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
 957 }
 958
 959 static struct mlxsw_sp_ipip_entry *
 960 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
 961                           enum mlxsw_sp_ipip_type ipipt,
 962                           struct net_device *ol_dev)
 963 {
 964         struct mlxsw_sp_ipip_entry *ipip_entry;
 965         struct mlxsw_sp_ipip_entry *ret = NULL;
 966
 967         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
 968         if (!ipip_entry)
 969                 return ERR_PTR(-ENOMEM);
 970
 971         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
 972                                                             ol_dev);
 973         if (IS_ERR(ipip_entry->ol_lb)) {
 974                 ret = ERR_CAST(ipip_entry->ol_lb);
 975                 goto err_ol_ipip_lb_create;
 976         }
 977
 978         ipip_entry->ipipt = ipipt;
 979         ipip_entry->ol_dev = ol_dev;
 980
 981         return ipip_entry;
 982
 983 err_ol_ipip_lb_create:
 984         kfree(ipip_entry);
 985         return ret;
 986 }
 987
 988 static void
 989 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp_ipip_entry *ipip_entry)
 990 {
 991         WARN_ON(ipip_entry->ref_count > 0);
 992         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
 993         kfree(ipip_entry);
 994 }
 995
 996 static __be32
 997 mlxsw_sp_ipip_netdev_saddr4(const struct net_device *ol_dev)
 998 {
 999         struct ip_tunnel *tun = netdev_priv(ol_dev);
1000
1001         return tun->parms.iph.saddr;
1002 }
1003
1004 union mlxsw_sp_l3addr
1005 mlxsw_sp_ipip_netdev_saddr(enum mlxsw_sp_l3proto proto,
1006                            const struct net_device *ol_dev)
1007 {
1008         switch (proto) {
1009         case MLXSW_SP_L3_PROTO_IPV4:
1010                 return (union mlxsw_sp_l3addr) {
1011                         .addr4 = mlxsw_sp_ipip_netdev_saddr4(ol_dev),
1012                 };
1013         case MLXSW_SP_L3_PROTO_IPV6:
1014                 break;
1015         };
1016
1017         WARN_ON(1);
1018         return (union mlxsw_sp_l3addr) {
1019                 .addr4 = 0,
1020         };
1021 }
1022
1023 __be32 mlxsw_sp_ipip_netdev_daddr4(const struct net_device *ol_dev)
1024 {
1025         struct ip_tunnel *tun = netdev_priv(ol_dev);
1026
1027         return tun->parms.iph.daddr;
1028 }
1029
1030 union mlxsw_sp_l3addr
1031 mlxsw_sp_ipip_netdev_daddr(enum mlxsw_sp_l3proto proto,
1032                            const struct net_device *ol_dev)
1033 {
1034         switch (proto) {
1035         case MLXSW_SP_L3_PROTO_IPV4:
1036                 return (union mlxsw_sp_l3addr) {
1037                         .addr4 = mlxsw_sp_ipip_netdev_daddr4(ol_dev),
1038                 };
1039         case MLXSW_SP_L3_PROTO_IPV6:
1040                 break;
1041         };
1042
1043         WARN_ON(1);
1044         return (union mlxsw_sp_l3addr) {
1045                 .addr4 = 0,
1046         };
1047 }
1048
1049 static bool mlxsw_sp_l3addr_eq(const union mlxsw_sp_l3addr *addr1,
1050                                const union mlxsw_sp_l3addr *addr2)
1051 {
1052         return !memcmp(addr1, addr2, sizeof(*addr1));
1053 }
1054
1055 static bool
1056 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1057                                   const enum mlxsw_sp_l3proto ul_proto,
1058                                   union mlxsw_sp_l3addr saddr,
1059                                   u32 ul_tb_id,
1060                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1061 {
1062         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1063         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1064         union mlxsw_sp_l3addr tun_saddr;
1065
1066         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1067                 return false;
1068
1069         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1070         return tun_ul_tb_id == ul_tb_id &&
1071                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1072 }
1073
1074 static int
1075 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1076                               struct mlxsw_sp_fib_entry *fib_entry,
1077                               struct mlxsw_sp_ipip_entry *ipip_entry)
1078 {
1079         u32 tunnel_index;
1080         int err;
1081
1082         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, 1, &tunnel_index);
1083         if (err)
1084                 return err;
1085
1086         ipip_entry->decap_fib_entry = fib_entry;
1087         fib_entry->decap.ipip_entry = ipip_entry;
1088         fib_entry->decap.tunnel_index = tunnel_index;
1089         return 0;
1090 }
1091
1092 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1093                                           struct mlxsw_sp_fib_entry *fib_entry)
1094 {
1095         /* Unlink this node from the IPIP entry that it's the decap entry of. */
1096         fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1097         fib_entry->decap.ipip_entry = NULL;
1098         mlxsw_sp_kvdl_free(mlxsw_sp, fib_entry->decap.tunnel_index);
1099 }
1100
1101 static struct mlxsw_sp_fib_node *
1102 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1103                          size_t addr_len, unsigned char prefix_len);
1104 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1105                                      struct mlxsw_sp_fib_entry *fib_entry);
1106
1107 static void
1108 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1109                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1110 {
1111         struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1112
1113         mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1114         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1115
1116         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1117 }
1118
1119 static void
1120 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1121                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1122                                   struct mlxsw_sp_fib_entry *decap_fib_entry)
1123 {
1124         if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1125                                           ipip_entry))
1126                 return;
1127         decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1128
1129         if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1130                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1131 }
1132
1133 /* Given an IPIP entry, find the corresponding decap route. */
1134 static struct mlxsw_sp_fib_entry *
1135 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1136                                struct mlxsw_sp_ipip_entry *ipip_entry)
1137 {
1138         static struct mlxsw_sp_fib_node *fib_node;
1139         const struct mlxsw_sp_ipip_ops *ipip_ops;
1140         struct mlxsw_sp_fib_entry *fib_entry;
1141         unsigned char saddr_prefix_len;
1142         union mlxsw_sp_l3addr saddr;
1143         struct mlxsw_sp_fib *ul_fib;
1144         struct mlxsw_sp_vr *ul_vr;
1145         const void *saddrp;
1146         size_t saddr_len;
1147         u32 ul_tb_id;
1148         u32 saddr4;
1149
1150         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1151
1152         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1153         ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1154         if (!ul_vr)
1155                 return NULL;
1156
1157         ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1158         saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1159                                            ipip_entry->ol_dev);
1160
1161         switch (ipip_ops->ul_proto) {
1162         case MLXSW_SP_L3_PROTO_IPV4:
1163                 saddr4 = be32_to_cpu(saddr.addr4);
1164                 saddrp = &saddr4;
1165                 saddr_len = 4;
1166                 saddr_prefix_len = 32;
1167                 break;
1168         case MLXSW_SP_L3_PROTO_IPV6:
1169                 WARN_ON(1);
1170                 return NULL;
1171         }
1172
1173         fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1174                                             saddr_prefix_len);
1175         if (!fib_node || list_empty(&fib_node->entry_list))
1176                 return NULL;
1177
1178         fib_entry = list_first_entry(&fib_node->entry_list,
1179                                      struct mlxsw_sp_fib_entry, list);
1180         if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1181                 return NULL;
1182
1183         return fib_entry;
1184 }
1185
1186 static struct mlxsw_sp_ipip_entry *
1187 mlxsw_sp_ipip_entry_get(struct mlxsw_sp *mlxsw_sp,
1188                         enum mlxsw_sp_ipip_type ipipt,
1189                         struct net_device *ol_dev)
1190 {
1191         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1192         struct mlxsw_sp_router *router = mlxsw_sp->router;
1193         struct mlxsw_sp_fib_entry *decap_fib_entry;
1194         struct mlxsw_sp_ipip_entry *ipip_entry;
1195         enum mlxsw_sp_l3proto ul_proto;
1196         union mlxsw_sp_l3addr saddr;
1197
1198         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1199                             ipip_list_node) {
1200                 if (ipip_entry->ol_dev == ol_dev)
1201                         goto inc_ref_count;
1202
1203                 /* The configuration where several tunnels have the same local
1204                  * address in the same underlay table needs special treatment in
1205                  * the HW. That is currently not implemented in the driver.
1206                  */
1207                 ul_proto = router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1208                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1209                 if (mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1210                                                       ul_tb_id, ipip_entry))
1211                         return ERR_PTR(-EEXIST);
1212         }
1213
1214         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1215         if (IS_ERR(ipip_entry))
1216                 return ipip_entry;
1217
1218         decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1219         if (decap_fib_entry)
1220                 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1221                                                   decap_fib_entry);
1222
1223         list_add_tail(&ipip_entry->ipip_list_node,
1224                       &mlxsw_sp->router->ipip_list);
1225
1226 inc_ref_count:
1227         ++ipip_entry->ref_count;
1228         return ipip_entry;
1229 }
1230
1231 static void
1232 mlxsw_sp_ipip_entry_put(struct mlxsw_sp *mlxsw_sp,
1233                         struct mlxsw_sp_ipip_entry *ipip_entry)
1234 {
1235         if (--ipip_entry->ref_count == 0) {
1236                 list_del(&ipip_entry->ipip_list_node);
1237                 if (ipip_entry->decap_fib_entry)
1238                         mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1239                 mlxsw_sp_ipip_entry_destroy(ipip_entry);
1240         }
1241 }
1242
1243 static bool
1244 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1245                                   const struct net_device *ul_dev,
1246                                   enum mlxsw_sp_l3proto ul_proto,
1247                                   union mlxsw_sp_l3addr ul_dip,
1248                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1249 {
1250         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1251         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1252         struct net_device *ipip_ul_dev;
1253
1254         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1255                 return false;
1256
1257         ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev);
1258         return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1259                                                  ul_tb_id, ipip_entry) &&
1260                (!ipip_ul_dev || ipip_ul_dev == ul_dev);
1261 }
1262
1263 /* Given decap parameters, find the corresponding IPIP entry. */
1264 static struct mlxsw_sp_ipip_entry *
1265 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp,
1266                                   const struct net_device *ul_dev,
1267                                   enum mlxsw_sp_l3proto ul_proto,
1268                                   union mlxsw_sp_l3addr ul_dip)
1269 {
1270         struct mlxsw_sp_ipip_entry *ipip_entry;
1271
1272         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1273                             ipip_list_node)
1274                 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1275                                                       ul_proto, ul_dip,
1276                                                       ipip_entry))
1277                         return ipip_entry;
1278
1279         return NULL;
1280 }
1281
1282 struct mlxsw_sp_neigh_key {
1283         struct neighbour *n;
1284 };
1285
1286 struct mlxsw_sp_neigh_entry {
1287         struct list_head rif_list_node;
1288         struct rhash_head ht_node;
1289         struct mlxsw_sp_neigh_key key;
1290         u16 rif;
1291         bool connected;
1292         unsigned char ha[ETH_ALEN];
1293         struct list_head nexthop_list; /* list of nexthops using
1294                                         * this neigh entry
1295                                         */
1296         struct list_head nexthop_neighs_list_node;
1297         unsigned int counter_index;
1298         bool counter_valid;
1299 };
1300
1301 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
1302         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
1303         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
1304         .key_len = sizeof(struct mlxsw_sp_neigh_key),
1305 };
1306
1307 struct mlxsw_sp_neigh_entry *
1308 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
1309                         struct mlxsw_sp_neigh_entry *neigh_entry)
1310 {
1311         if (!neigh_entry) {
1312                 if (list_empty(&rif->neigh_list))
1313                         return NULL;
1314                 else
1315                         return list_first_entry(&rif->neigh_list,
1316                                                 typeof(*neigh_entry),
1317                                                 rif_list_node);
1318         }
1319         if (neigh_entry->rif_list_node.next == &rif->neigh_list)
1320                 return NULL;
1321         return list_next_entry(neigh_entry, rif_list_node);
1322 }
1323
1324 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
1325 {
1326         return neigh_entry->key.n->tbl->family;
1327 }
1328
1329 unsigned char *
1330 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
1331 {
1332         return neigh_entry->ha;
1333 }
1334
1335 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1336 {
1337         struct neighbour *n;
1338
1339         n = neigh_entry->key.n;
1340         return ntohl(*((__be32 *) n->primary_key));
1341 }
1342
1343 struct in6_addr *
1344 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1345 {
1346         struct neighbour *n;
1347
1348         n = neigh_entry->key.n;
1349         return (struct in6_addr *) &n->primary_key;
1350 }
1351
1352 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
1353                                struct mlxsw_sp_neigh_entry *neigh_entry,
1354                                u64 *p_counter)
1355 {
1356         if (!neigh_entry->counter_valid)
1357                 return -EINVAL;
1358
1359         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
1360                                          p_counter, NULL);
1361 }
1362
1363 static struct mlxsw_sp_neigh_entry *
1364 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
1365                            u16 rif)
1366 {
1367         struct mlxsw_sp_neigh_entry *neigh_entry;
1368
1369         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
1370         if (!neigh_entry)
1371                 return NULL;
1372
1373         neigh_entry->key.n = n;
1374         neigh_entry->rif = rif;
1375         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
1376
1377         return neigh_entry;
1378 }
1379
1380 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
1381 {
1382         kfree(neigh_entry);
1383 }
1384
1385 static int
1386 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
1387                             struct mlxsw_sp_neigh_entry *neigh_entry)
1388 {
1389         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
1390                                       &neigh_entry->ht_node,
1391                                       mlxsw_sp_neigh_ht_params);
1392 }
1393
1394 static void
1395 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
1396                             struct mlxsw_sp_neigh_entry *neigh_entry)
1397 {
1398         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
1399                                &neigh_entry->ht_node,
1400                                mlxsw_sp_neigh_ht_params);
1401 }
1402
1403 static bool
1404 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
1405                                     struct mlxsw_sp_neigh_entry *neigh_entry)
1406 {
1407         struct devlink *devlink;
1408         const char *table_name;
1409
1410         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
1411         case AF_INET:
1412                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
1413                 break;
1414         case AF_INET6:
1415                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
1416                 break;
1417         default:
1418                 WARN_ON(1);
1419                 return false;
1420         }
1421
1422         devlink = priv_to_devlink(mlxsw_sp->core);
1423         return devlink_dpipe_table_counter_enabled(devlink, table_name);
1424 }
1425
1426 static void
1427 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
1428                              struct mlxsw_sp_neigh_entry *neigh_entry)
1429 {
1430         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
1431                 return;
1432
1433         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
1434                 return;
1435
1436         neigh_entry->counter_valid = true;
1437 }
1438
1439 static void
1440 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
1441                             struct mlxsw_sp_neigh_entry *neigh_entry)
1442 {
1443         if (!neigh_entry->counter_valid)
1444                 return;
1445         mlxsw_sp_flow_counter_free(mlxsw_sp,
1446                                    neigh_entry->counter_index);
1447         neigh_entry->counter_valid = false;
1448 }
1449
1450 static struct mlxsw_sp_neigh_entry *
1451 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1452 {
1453         struct mlxsw_sp_neigh_entry *neigh_entry;
1454         struct mlxsw_sp_rif *rif;
1455         int err;
1456
1457         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
1458         if (!rif)
1459                 return ERR_PTR(-EINVAL);
1460
1461         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
1462         if (!neigh_entry)
1463                 return ERR_PTR(-ENOMEM);
1464
1465         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
1466         if (err)
1467                 goto err_neigh_entry_insert;
1468
1469         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1470         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
1471
1472         return neigh_entry;
1473
1474 err_neigh_entry_insert:
1475         mlxsw_sp_neigh_entry_free(neigh_entry);
1476         return ERR_PTR(err);
1477 }
1478
1479 static void
1480 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1481                              struct mlxsw_sp_neigh_entry *neigh_entry)
1482 {
1483         list_del(&neigh_entry->rif_list_node);
1484         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1485         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
1486         mlxsw_sp_neigh_entry_free(neigh_entry);
1487 }
1488
1489 static struct mlxsw_sp_neigh_entry *
1490 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1491 {
1492         struct mlxsw_sp_neigh_key key;
1493
1494         key.n = n;
1495         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
1496                                       &key, mlxsw_sp_neigh_ht_params);
1497 }
1498
1499 static void
1500 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
1501 {
1502         unsigned long interval;
1503
1504 #if IS_ENABLED(CONFIG_IPV6)
1505         interval = min_t(unsigned long,
1506                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
1507                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
1508 #else
1509         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
1510 #endif
1511         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
1512 }
1513
1514 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1515                                                    char *rauhtd_pl,
1516                                                    int ent_index)
1517 {
1518         struct net_device *dev;
1519         struct neighbour *n;
1520         __be32 dipn;
1521         u32 dip;
1522         u16 rif;
1523
1524         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
1525
1526         if (!mlxsw_sp->router->rifs[rif]) {
1527                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1528                 return;
1529         }
1530
1531         dipn = htonl(dip);
1532         dev = mlxsw_sp->router->rifs[rif]->dev;
1533         n = neigh_lookup(&arp_tbl, &dipn, dev);
1534         if (!n) {
1535                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
1536                            &dip);
1537                 return;
1538         }
1539
1540         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
1541         neigh_event_send(n, NULL);
1542         neigh_release(n);
1543 }
1544
1545 #if IS_ENABLED(CONFIG_IPV6)
1546 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1547                                                    char *rauhtd_pl,
1548                                                    int rec_index)
1549 {
1550         struct net_device *dev;
1551         struct neighbour *n;
1552         struct in6_addr dip;
1553         u16 rif;
1554
1555         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
1556                                          (char *) &dip);
1557
1558         if (!mlxsw_sp->router->rifs[rif]) {
1559                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1560                 return;
1561         }
1562
1563         dev = mlxsw_sp->router->rifs[rif]->dev;
1564         n = neigh_lookup(&nd_tbl, &dip, dev);
1565         if (!n) {
1566                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
1567                            &dip);
1568                 return;
1569         }
1570
1571         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1572         neigh_event_send(n, NULL);
1573         neigh_release(n);
1574 }
1575 #else
1576 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1577                                                    char *rauhtd_pl,
1578                                                    int rec_index)
1579 {
1580 }
1581 #endif
1582
1583 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1584                                                    char *rauhtd_pl,
1585                                                    int rec_index)
1586 {
1587         u8 num_entries;
1588         int i;
1589
1590         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1591                                                                 rec_index);
1592         /* Hardware starts counting at 0, so add 1. */
1593         num_entries++;
1594
1595         /* Each record consists of several neighbour entries. */
1596         for (i = 0; i < num_entries; i++) {
1597                 int ent_index;
1598
1599                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1600                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1601                                                        ent_index);
1602         }
1603
1604 }
1605
1606 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1607                                                    char *rauhtd_pl,
1608                                                    int rec_index)
1609 {
1610         /* One record contains one entry. */
1611         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1612                                                rec_index);
1613 }
1614
1615 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1616                                               char *rauhtd_pl, int rec_index)
1617 {
1618         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1619         case MLXSW_REG_RAUHTD_TYPE_IPV4:
1620                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1621                                                        rec_index);
1622                 break;
1623         case MLXSW_REG_RAUHTD_TYPE_IPV6:
1624                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1625                                                        rec_index);
1626                 break;
1627         }
1628 }
1629
1630 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1631 {
1632         u8 num_rec, last_rec_index, num_entries;
1633
1634         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1635         last_rec_index = num_rec - 1;
1636
1637         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1638                 return false;
1639         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1640             MLXSW_REG_RAUHTD_TYPE_IPV6)
1641                 return true;
1642
1643         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1644                                                                 last_rec_index);
1645         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1646                 return true;
1647         return false;
1648 }
1649
1650 static int
1651 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1652                                        char *rauhtd_pl,
1653                                        enum mlxsw_reg_rauhtd_type type)
1654 {
1655         int i, num_rec;
1656         int err;
1657
1658         /* Make sure the neighbour's netdev isn't removed in the
1659          * process.
1660          */
1661         rtnl_lock();
1662         do {
1663                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
1664                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1665                                       rauhtd_pl);
1666                 if (err) {
1667                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1668                         break;
1669                 }
1670                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1671                 for (i = 0; i < num_rec; i++)
1672                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1673                                                           i);
1674         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1675         rtnl_unlock();
1676
1677         return err;
1678 }
1679
1680 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1681 {
1682         enum mlxsw_reg_rauhtd_type type;
1683         char *rauhtd_pl;
1684         int err;
1685
1686         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1687         if (!rauhtd_pl)
1688                 return -ENOMEM;
1689
1690         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1691         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1692         if (err)
1693                 goto out;
1694
1695         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1696         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1697 out:
1698         kfree(rauhtd_pl);
1699         return err;
1700 }
1701
1702 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1703 {
1704         struct mlxsw_sp_neigh_entry *neigh_entry;
1705
1706         /* Take RTNL mutex here to prevent lists from changes */
1707         rtnl_lock();
1708         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1709                             nexthop_neighs_list_node)
1710                 /* If this neigh have nexthops, make the kernel think this neigh
1711                  * is active regardless of the traffic.
1712                  */
1713                 neigh_event_send(neigh_entry->key.n, NULL);
1714         rtnl_unlock();
1715 }
1716
1717 static void
1718 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1719 {
1720         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1721
1722         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1723                                msecs_to_jiffies(interval));
1724 }
1725
1726 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1727 {
1728         struct mlxsw_sp_router *router;
1729         int err;
1730
1731         router = container_of(work, struct mlxsw_sp_router,
1732                               neighs_update.dw.work);
1733         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1734         if (err)
1735                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1736
1737         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1738
1739         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1740 }
1741
1742 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1743 {
1744         struct mlxsw_sp_neigh_entry *neigh_entry;
1745         struct mlxsw_sp_router *router;
1746
1747         router = container_of(work, struct mlxsw_sp_router,
1748                               nexthop_probe_dw.work);
1749         /* Iterate over nexthop neighbours, find those who are unresolved and
1750          * send arp on them. This solves the chicken-egg problem when
1751          * the nexthop wouldn't get offloaded until the neighbor is resolved
1752          * but it wouldn't get resolved ever in case traffic is flowing in HW
1753          * using different nexthop.
1754          *
1755          * Take RTNL mutex here to prevent lists from changes.
1756          */
1757         rtnl_lock();
1758         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1759                             nexthop_neighs_list_node)
1760                 if (!neigh_entry->connected)
1761                         neigh_event_send(neigh_entry->key.n, NULL);
1762         rtnl_unlock();
1763
1764         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1765                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1766 }
1767
1768 static void
1769 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1770                               struct mlxsw_sp_neigh_entry *neigh_entry,
1771                               bool removing);
1772
1773 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1774 {
1775         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1776                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1777 }
1778
1779 static void
1780 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1781                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1782                                 enum mlxsw_reg_rauht_op op)
1783 {
1784         struct neighbour *n = neigh_entry->key.n;
1785         u32 dip = ntohl(*((__be32 *) n->primary_key));
1786         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1787
1788         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1789                               dip);
1790         if (neigh_entry->counter_valid)
1791                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1792                                              neigh_entry->counter_index);
1793         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1794 }
1795
1796 static void
1797 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1798                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1799                                 enum mlxsw_reg_rauht_op op)
1800 {
1801         struct neighbour *n = neigh_entry->key.n;
1802         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1803         const char *dip = n->primary_key;
1804
1805         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1806                               dip);
1807         if (neigh_entry->counter_valid)
1808                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1809                                              neigh_entry->counter_index);
1810         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1811 }
1812
1813 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
1814 {
1815         struct neighbour *n = neigh_entry->key.n;
1816
1817         /* Packets with a link-local destination address are trapped
1818          * after LPM lookup and never reach the neighbour table, so
1819          * there is no need to program such neighbours to the device.
1820          */
1821         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1822             IPV6_ADDR_LINKLOCAL)
1823                 return true;
1824         return false;
1825 }
1826
1827 static void
1828 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1829                             struct mlxsw_sp_neigh_entry *neigh_entry,
1830                             bool adding)
1831 {
1832         if (!adding && !neigh_entry->connected)
1833                 return;
1834         neigh_entry->connected = adding;
1835         if (neigh_entry->key.n->tbl->family == AF_INET) {
1836                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1837                                                 mlxsw_sp_rauht_op(adding));
1838         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
1839                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
1840                         return;
1841                 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1842                                                 mlxsw_sp_rauht_op(adding));
1843         } else {
1844                 WARN_ON_ONCE(1);
1845         }
1846 }
1847
1848 void
1849 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
1850                                     struct mlxsw_sp_neigh_entry *neigh_entry,
1851                                     bool adding)
1852 {
1853         if (adding)
1854                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1855         else
1856                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1857         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
1858 }
1859
1860 struct mlxsw_sp_neigh_event_work {
1861         struct work_struct work;
1862         struct mlxsw_sp *mlxsw_sp;
1863         struct neighbour *n;
1864 };
1865
1866 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1867 {
1868         struct mlxsw_sp_neigh_event_work *neigh_work =
1869                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1870         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1871         struct mlxsw_sp_neigh_entry *neigh_entry;
1872         struct neighbour *n = neigh_work->n;
1873         unsigned char ha[ETH_ALEN];
1874         bool entry_connected;
1875         u8 nud_state, dead;
1876
1877         /* If these parameters are changed after we release the lock,
1878          * then we are guaranteed to receive another event letting us
1879          * know about it.
1880          */
1881         read_lock_bh(&n->lock);
1882         memcpy(ha, n->ha, ETH_ALEN);
1883         nud_state = n->nud_state;
1884         dead = n->dead;
1885         read_unlock_bh(&n->lock);
1886
1887         rtnl_lock();
1888         entry_connected = nud_state & NUD_VALID && !dead;
1889         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1890         if (!entry_connected && !neigh_entry)
1891                 goto out;
1892         if (!neigh_entry) {
1893                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1894                 if (IS_ERR(neigh_entry))
1895                         goto out;
1896         }
1897
1898         memcpy(neigh_entry->ha, ha, ETH_ALEN);
1899         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1900         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1901
1902         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1903                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1904
1905 out:
1906         rtnl_unlock();
1907         neigh_release(n);
1908         kfree(neigh_work);
1909 }
1910
1911 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1912                                    unsigned long event, void *ptr)
1913 {
1914         struct mlxsw_sp_neigh_event_work *neigh_work;
1915         struct mlxsw_sp_port *mlxsw_sp_port;
1916         struct mlxsw_sp *mlxsw_sp;
1917         unsigned long interval;
1918         struct neigh_parms *p;
1919         struct neighbour *n;
1920
1921         switch (event) {
1922         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1923                 p = ptr;
1924
1925                 /* We don't care about changes in the default table. */
1926                 if (!p->dev || (p->tbl->family != AF_INET &&
1927                                 p->tbl->family != AF_INET6))
1928                         return NOTIFY_DONE;
1929
1930                 /* We are in atomic context and can't take RTNL mutex,
1931                  * so use RCU variant to walk the device chain.
1932                  */
1933                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1934                 if (!mlxsw_sp_port)
1935                         return NOTIFY_DONE;
1936
1937                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1938                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1939                 mlxsw_sp->router->neighs_update.interval = interval;
1940
1941                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1942                 break;
1943         case NETEVENT_NEIGH_UPDATE:
1944                 n = ptr;
1945
1946                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
1947                         return NOTIFY_DONE;
1948
1949                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1950                 if (!mlxsw_sp_port)
1951                         return NOTIFY_DONE;
1952
1953                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1954                 if (!neigh_work) {
1955                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1956                         return NOTIFY_BAD;
1957                 }
1958
1959                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1960                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1961                 neigh_work->n = n;
1962
1963                 /* Take a reference to ensure the neighbour won't be
1964                  * destructed until we drop the reference in delayed
1965                  * work.
1966                  */
1967                 neigh_clone(n);
1968                 mlxsw_core_schedule_work(&neigh_work->work);
1969                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1970                 break;
1971         }
1972
1973         return NOTIFY_DONE;
1974 }
1975
1976 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1977 {
1978         int err;
1979
1980         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1981                               &mlxsw_sp_neigh_ht_params);
1982         if (err)
1983                 return err;
1984
1985         /* Initialize the polling interval according to the default
1986          * table.
1987          */
1988         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1989
1990         /* Create the delayed works for the activity_update */
1991         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1992                           mlxsw_sp_router_neighs_update_work);
1993         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1994                           mlxsw_sp_router_probe_unresolved_nexthops);
1995         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1996         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1997         return 0;
1998 }
1999
2000 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2001 {
2002         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2003         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2004         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2005 }
2006
2007 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2008                                          struct mlxsw_sp_rif *rif)
2009 {
2010         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2011
2012         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2013                                  rif_list_node) {
2014                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2015                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2016         }
2017 }
2018
2019 enum mlxsw_sp_nexthop_type {
2020         MLXSW_SP_NEXTHOP_TYPE_ETH,
2021         MLXSW_SP_NEXTHOP_TYPE_IPIP,
2022 };
2023
2024 struct mlxsw_sp_nexthop_key {
2025         struct fib_nh *fib_nh;
2026 };
2027
2028 struct mlxsw_sp_nexthop {
2029         struct list_head neigh_list_node; /* member of neigh entry list */
2030         struct list_head rif_list_node;
2031         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
2032                                                 * this belongs to
2033                                                 */
2034         struct rhash_head ht_node;
2035         struct mlxsw_sp_nexthop_key key;
2036         unsigned char gw_addr[sizeof(struct in6_addr)];
2037         int ifindex;
2038         struct mlxsw_sp_rif *rif;
2039         u8 should_offload:1, /* set indicates this neigh is connected and
2040                               * should be put to KVD linear area of this group.
2041                               */
2042            offloaded:1, /* set in case the neigh is actually put into
2043                          * KVD linear area of this group.
2044                          */
2045            update:1; /* set indicates that MAC of this neigh should be
2046                       * updated in HW
2047                       */
2048         enum mlxsw_sp_nexthop_type type;
2049         union {
2050                 struct mlxsw_sp_neigh_entry *neigh_entry;
2051                 struct mlxsw_sp_ipip_entry *ipip_entry;
2052         };
2053 };
2054
2055 struct mlxsw_sp_nexthop_group {
2056         void *priv;
2057         struct rhash_head ht_node;
2058         struct list_head fib_list; /* list of fib entries that use this group */
2059         struct neigh_table *neigh_tbl;
2060         u8 adj_index_valid:1,
2061            gateway:1; /* routes using the group use a gateway */
2062         u32 adj_index;
2063         u16 ecmp_size;
2064         u16 count;
2065         struct mlxsw_sp_nexthop nexthops[0];
2066 #define nh_rif  nexthops[0].rif
2067 };
2068
2069 static struct fib_info *
2070 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
2071 {
2072         return nh_grp->priv;
2073 }
2074
2075 struct mlxsw_sp_nexthop_group_cmp_arg {
2076         enum mlxsw_sp_l3proto proto;
2077         union {
2078                 struct fib_info *fi;
2079                 struct mlxsw_sp_fib6_entry *fib6_entry;
2080         };
2081 };
2082
2083 static bool
2084 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
2085                                     const struct in6_addr *gw, int ifindex)
2086 {
2087         int i;
2088
2089         for (i = 0; i < nh_grp->count; i++) {
2090                 const struct mlxsw_sp_nexthop *nh;
2091
2092                 nh = &nh_grp->nexthops[i];
2093                 if (nh->ifindex == ifindex &&
2094                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
2095                         return true;
2096         }
2097
2098         return false;
2099 }
2100
2101 static bool
2102 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
2103                             const struct mlxsw_sp_fib6_entry *fib6_entry)
2104 {
2105         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2106
2107         if (nh_grp->count != fib6_entry->nrt6)
2108                 return false;
2109
2110         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2111                 struct in6_addr *gw;
2112                 int ifindex;
2113
2114                 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
2115                 gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
2116                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
2117                         return false;
2118         }
2119
2120         return true;
2121 }
2122
2123 static int
2124 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
2125 {
2126         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
2127         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
2128
2129         switch (cmp_arg->proto) {
2130         case MLXSW_SP_L3_PROTO_IPV4:
2131                 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
2132         case MLXSW_SP_L3_PROTO_IPV6:
2133                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
2134                                                     cmp_arg->fib6_entry);
2135         default:
2136                 WARN_ON(1);
2137                 return 1;
2138         }
2139 }
2140
2141 static int
2142 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
2143 {
2144         return nh_grp->neigh_tbl->family;
2145 }
2146
2147 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
2148 {
2149         const struct mlxsw_sp_nexthop_group *nh_grp = data;
2150         const struct mlxsw_sp_nexthop *nh;
2151         struct fib_info *fi;
2152         unsigned int val;
2153         int i;
2154
2155         switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
2156         case AF_INET:
2157                 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
2158                 return jhash(&fi, sizeof(fi), seed);
2159         case AF_INET6:
2160                 val = nh_grp->count;
2161                 for (i = 0; i < nh_grp->count; i++) {
2162                         nh = &nh_grp->nexthops[i];
2163                         val ^= nh->ifindex;
2164                 }
2165                 return jhash(&val, sizeof(val), seed);
2166         default:
2167                 WARN_ON(1);
2168                 return 0;
2169         }
2170 }
2171
2172 static u32
2173 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
2174 {
2175         unsigned int val = fib6_entry->nrt6;
2176         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2177         struct net_device *dev;
2178
2179         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2180                 dev = mlxsw_sp_rt6->rt->dst.dev;
2181                 val ^= dev->ifindex;
2182         }
2183
2184         return jhash(&val, sizeof(val), seed);
2185 }
2186
2187 static u32
2188 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
2189 {
2190         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
2191
2192         switch (cmp_arg->proto) {
2193         case MLXSW_SP_L3_PROTO_IPV4:
2194                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
2195         case MLXSW_SP_L3_PROTO_IPV6:
2196                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
2197         default:
2198                 WARN_ON(1);
2199                 return 0;
2200         }
2201 }
2202
2203 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
2204         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
2205         .hashfn      = mlxsw_sp_nexthop_group_hash,
2206         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
2207         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
2208 };
2209
2210 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
2211                                          struct mlxsw_sp_nexthop_group *nh_grp)
2212 {
2213         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2214             !nh_grp->gateway)
2215                 return 0;
2216
2217         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
2218                                       &nh_grp->ht_node,
2219                                       mlxsw_sp_nexthop_group_ht_params);
2220 }
2221
2222 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
2223                                           struct mlxsw_sp_nexthop_group *nh_grp)
2224 {
2225         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2226             !nh_grp->gateway)
2227                 return;
2228
2229         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
2230                                &nh_grp->ht_node,
2231                                mlxsw_sp_nexthop_group_ht_params);
2232 }
2233
2234 static struct mlxsw_sp_nexthop_group *
2235 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
2236                                struct fib_info *fi)
2237 {
2238         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2239
2240         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
2241         cmp_arg.fi = fi;
2242         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2243                                       &cmp_arg,
2244                                       mlxsw_sp_nexthop_group_ht_params);
2245 }
2246
2247 static struct mlxsw_sp_nexthop_group *
2248 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
2249                                struct mlxsw_sp_fib6_entry *fib6_entry)
2250 {
2251         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2252
2253         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
2254         cmp_arg.fib6_entry = fib6_entry;
2255         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2256                                       &cmp_arg,
2257                                       mlxsw_sp_nexthop_group_ht_params);
2258 }
2259
2260 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
2261         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
2262         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
2263         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
2264 };
2265
2266 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
2267                                    struct mlxsw_sp_nexthop *nh)
2268 {
2269         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
2270                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
2271 }
2272
2273 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
2274                                     struct mlxsw_sp_nexthop *nh)
2275 {
2276         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
2277                                mlxsw_sp_nexthop_ht_params);
2278 }
2279
2280 static struct mlxsw_sp_nexthop *
2281 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
2282                         struct mlxsw_sp_nexthop_key key)
2283 {
2284         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
2285                                       mlxsw_sp_nexthop_ht_params);
2286 }
2287
2288 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
2289                                              const struct mlxsw_sp_fib *fib,
2290                                              u32 adj_index, u16 ecmp_size,
2291                                              u32 new_adj_index,
2292                                              u16 new_ecmp_size)
2293 {
2294         char raleu_pl[MLXSW_REG_RALEU_LEN];
2295
2296         mlxsw_reg_raleu_pack(raleu_pl,
2297                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
2298                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
2299                              new_ecmp_size);
2300         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
2301 }
2302
2303 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
2304                                           struct mlxsw_sp_nexthop_group *nh_grp,
2305                                           u32 old_adj_index, u16 old_ecmp_size)
2306 {
2307         struct mlxsw_sp_fib_entry *fib_entry;
2308         struct mlxsw_sp_fib *fib = NULL;
2309         int err;
2310
2311         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2312                 if (fib == fib_entry->fib_node->fib)
2313                         continue;
2314                 fib = fib_entry->fib_node->fib;
2315                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
2316                                                         old_adj_index,
2317                                                         old_ecmp_size,
2318                                                         nh_grp->adj_index,
2319                                                         nh_grp->ecmp_size);
2320                 if (err)
2321                         return err;
2322         }
2323         return 0;
2324 }
2325
2326 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
2327                                        struct mlxsw_sp_nexthop *nh)
2328 {
2329         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2330         char ratr_pl[MLXSW_REG_RATR_LEN];
2331
2332         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
2333                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
2334                             adj_index, neigh_entry->rif);
2335         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
2336         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
2337 }
2338
2339 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
2340                                         u32 adj_index,
2341                                         struct mlxsw_sp_nexthop *nh)
2342 {
2343         const struct mlxsw_sp_ipip_ops *ipip_ops;
2344
2345         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
2346         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
2347 }
2348
2349 static int
2350 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
2351                               struct mlxsw_sp_nexthop_group *nh_grp,
2352                               bool reallocate)
2353 {
2354         u32 adj_index = nh_grp->adj_index; /* base */
2355         struct mlxsw_sp_nexthop *nh;
2356         int i;
2357         int err;
2358
2359         for (i = 0; i < nh_grp->count; i++) {
2360                 nh = &nh_grp->nexthops[i];
2361
2362                 if (!nh->should_offload) {
2363                         nh->offloaded = 0;
2364                         continue;
2365                 }
2366
2367                 if (nh->update || reallocate) {
2368                         switch (nh->type) {
2369                         case MLXSW_SP_NEXTHOP_TYPE_ETH:
2370                                 err = mlxsw_sp_nexthop_mac_update
2371                                             (mlxsw_sp, adj_index, nh);
2372                                 break;
2373                         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2374                                 err = mlxsw_sp_nexthop_ipip_update
2375                                             (mlxsw_sp, adj_index, nh);
2376                                 break;
2377                         }
2378                         if (err)
2379                                 return err;
2380                         nh->update = 0;
2381                         nh->offloaded = 1;
2382                 }
2383                 adj_index++;
2384         }
2385         return 0;
2386 }
2387
2388 static bool
2389 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2390                                  const struct mlxsw_sp_fib_entry *fib_entry);
2391
2392 static int
2393 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
2394                                     struct mlxsw_sp_nexthop_group *nh_grp)
2395 {
2396         struct mlxsw_sp_fib_entry *fib_entry;
2397         int err;
2398
2399         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2400                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2401                                                       fib_entry))
2402                         continue;
2403                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2404                 if (err)
2405                         return err;
2406         }
2407         return 0;
2408 }
2409
2410 static void
2411 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2412                                    enum mlxsw_reg_ralue_op op, int err);
2413
2414 static void
2415 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
2416 {
2417         enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
2418         struct mlxsw_sp_fib_entry *fib_entry;
2419
2420         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2421                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2422                                                       fib_entry))
2423                         continue;
2424                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2425         }
2426 }
2427
2428 static void
2429 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
2430                                struct mlxsw_sp_nexthop_group *nh_grp)
2431 {
2432         struct mlxsw_sp_nexthop *nh;
2433         bool offload_change = false;
2434         u32 adj_index;
2435         u16 ecmp_size = 0;
2436         bool old_adj_index_valid;
2437         u32 old_adj_index;
2438         u16 old_ecmp_size;
2439         int i;
2440         int err;
2441
2442         if (!nh_grp->gateway) {
2443                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2444                 return;
2445         }
2446
2447         for (i = 0; i < nh_grp->count; i++) {
2448                 nh = &nh_grp->nexthops[i];
2449
2450                 if (nh->should_offload != nh->offloaded) {
2451                         offload_change = true;
2452                         if (nh->should_offload)
2453                                 nh->update = 1;
2454                 }
2455                 if (nh->should_offload)
2456                         ecmp_size++;
2457         }
2458         if (!offload_change) {
2459                 /* Nothing was added or removed, so no need to reallocate. Just
2460                  * update MAC on existing adjacency indexes.
2461                  */
2462                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false);
2463                 if (err) {
2464                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2465                         goto set_trap;
2466                 }
2467                 return;
2468         }
2469         if (!ecmp_size)
2470                 /* No neigh of this group is connected so we just set
2471                  * the trap and let everthing flow through kernel.
2472                  */
2473                 goto set_trap;
2474
2475         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
2476         if (err) {
2477                 /* We ran out of KVD linear space, just set the
2478                  * trap and let everything flow through kernel.
2479                  */
2480                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
2481                 goto set_trap;
2482         }
2483         old_adj_index_valid = nh_grp->adj_index_valid;
2484         old_adj_index = nh_grp->adj_index;
2485         old_ecmp_size = nh_grp->ecmp_size;
2486         nh_grp->adj_index_valid = 1;
2487         nh_grp->adj_index = adj_index;
2488         nh_grp->ecmp_size = ecmp_size;
2489         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true);
2490         if (err) {
2491                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2492                 goto set_trap;
2493         }
2494
2495         if (!old_adj_index_valid) {
2496                 /* The trap was set for fib entries, so we have to call
2497                  * fib entry update to unset it and use adjacency index.
2498                  */
2499                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2500                 if (err) {
2501                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
2502                         goto set_trap;
2503                 }
2504                 return;
2505         }
2506
2507         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
2508                                              old_adj_index, old_ecmp_size);
2509         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
2510         if (err) {
2511                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
2512                 goto set_trap;
2513         }
2514
2515         /* Offload state within the group changed, so update the flags. */
2516         mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
2517
2518         return;
2519
2520 set_trap:
2521         old_adj_index_valid = nh_grp->adj_index_valid;
2522         nh_grp->adj_index_valid = 0;
2523         for (i = 0; i < nh_grp->count; i++) {
2524                 nh = &nh_grp->nexthops[i];
2525                 nh->offloaded = 0;
2526         }
2527         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2528         if (err)
2529                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
2530         if (old_adj_index_valid)
2531                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
2532 }
2533
2534 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
2535                                             bool removing)
2536 {
2537         if (!removing)
2538                 nh->should_offload = 1;
2539         else if (nh->offloaded)
2540                 nh->should_offload = 0;
2541         nh->update = 1;
2542 }
2543
2544 static void
2545 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2546                               struct mlxsw_sp_neigh_entry *neigh_entry,
2547                               bool removing)
2548 {
2549         struct mlxsw_sp_nexthop *nh;
2550
2551         list_for_each_entry(nh, &neigh_entry->nexthop_list,
2552                             neigh_list_node) {
2553                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
2554                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2555         }
2556 }
2557
2558 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
2559                                       struct mlxsw_sp_rif *rif)
2560 {
2561         if (nh->rif)
2562                 return;
2563
2564         nh->rif = rif;
2565         list_add(&nh->rif_list_node, &rif->nexthop_list);
2566 }
2567
2568 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
2569 {
2570         if (!nh->rif)
2571                 return;
2572
2573         list_del(&nh->rif_list_node);
2574         nh->rif = NULL;
2575 }
2576
2577 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
2578                                        struct mlxsw_sp_nexthop *nh)
2579 {
2580         struct mlxsw_sp_neigh_entry *neigh_entry;
2581         struct neighbour *n;
2582         u8 nud_state, dead;
2583         int err;
2584
2585         if (!nh->nh_grp->gateway || nh->neigh_entry)
2586                 return 0;
2587
2588         /* Take a reference of neigh here ensuring that neigh would
2589          * not be destructed before the nexthop entry is finished.
2590          * The reference is taken either in neigh_lookup() or
2591          * in neigh_create() in case n is not found.
2592          */
2593         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
2594         if (!n) {
2595                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
2596                                  nh->rif->dev);
2597                 if (IS_ERR(n))
2598                         return PTR_ERR(n);
2599                 neigh_event_send(n, NULL);
2600         }
2601         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2602         if (!neigh_entry) {
2603                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2604                 if (IS_ERR(neigh_entry)) {
2605                         err = -EINVAL;
2606                         goto err_neigh_entry_create;
2607                 }
2608         }
2609
2610         /* If that is the first nexthop connected to that neigh, add to
2611          * nexthop_neighs_list
2612          */
2613         if (list_empty(&neigh_entry->nexthop_list))
2614                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
2615                               &mlxsw_sp->router->nexthop_neighs_list);
2616
2617         nh->neigh_entry = neigh_entry;
2618         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
2619         read_lock_bh(&n->lock);
2620         nud_state = n->nud_state;
2621         dead = n->dead;
2622         read_unlock_bh(&n->lock);
2623         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
2624
2625         return 0;
2626
2627 err_neigh_entry_create:
2628         neigh_release(n);
2629         return err;
2630 }
2631
2632 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
2633                                         struct mlxsw_sp_nexthop *nh)
2634 {
2635         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2636         struct neighbour *n;
2637
2638         if (!neigh_entry)
2639                 return;
2640         n = neigh_entry->key.n;
2641
2642         __mlxsw_sp_nexthop_neigh_update(nh, true);
2643         list_del(&nh->neigh_list_node);
2644         nh->neigh_entry = NULL;
2645
2646         /* If that is the last nexthop connected to that neigh, remove from
2647          * nexthop_neighs_list
2648          */
2649         if (list_empty(&neigh_entry->nexthop_list))
2650                 list_del(&neigh_entry->nexthop_neighs_list_node);
2651
2652         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2653                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2654
2655         neigh_release(n);
2656 }
2657
2658 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2659                                       const struct net_device *dev,
2660                                       enum mlxsw_sp_ipip_type *p_type)
2661 {
2662         struct mlxsw_sp_router *router = mlxsw_sp->router;
2663         const struct mlxsw_sp_ipip_ops *ipip_ops;
2664         enum mlxsw_sp_ipip_type ipipt;
2665
2666         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
2667                 ipip_ops = router->ipip_ops_arr[ipipt];
2668                 if (dev->type == ipip_ops->dev_type) {
2669                         if (p_type)
2670                                 *p_type = ipipt;
2671                         return true;
2672                 }
2673         }
2674         return false;
2675 }
2676
2677 static int mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
2678                                       enum mlxsw_sp_ipip_type ipipt,
2679                                       struct mlxsw_sp_nexthop *nh,
2680                                       struct net_device *ol_dev)
2681 {
2682         if (!nh->nh_grp->gateway || nh->ipip_entry)
2683                 return 0;
2684
2685         nh->ipip_entry = mlxsw_sp_ipip_entry_get(mlxsw_sp, ipipt, ol_dev);
2686         if (IS_ERR(nh->ipip_entry))
2687                 return PTR_ERR(nh->ipip_entry);
2688
2689         __mlxsw_sp_nexthop_neigh_update(nh, false);
2690         return 0;
2691 }
2692
2693 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
2694                                        struct mlxsw_sp_nexthop *nh)
2695 {
2696         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
2697
2698         if (!ipip_entry)
2699                 return;
2700
2701         __mlxsw_sp_nexthop_neigh_update(nh, true);
2702         mlxsw_sp_ipip_entry_put(mlxsw_sp, ipip_entry);
2703         nh->ipip_entry = NULL;
2704 }
2705
2706 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2707                                         const struct fib_nh *fib_nh,
2708                                         enum mlxsw_sp_ipip_type *p_ipipt)
2709 {
2710         struct net_device *dev = fib_nh->nh_dev;
2711
2712         return dev &&
2713                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
2714                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
2715 }
2716
2717 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
2718                                        struct mlxsw_sp_nexthop *nh)
2719 {
2720         switch (nh->type) {
2721         case MLXSW_SP_NEXTHOP_TYPE_ETH:
2722                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2723                 mlxsw_sp_nexthop_rif_fini(nh);
2724                 break;
2725         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2726                 mlxsw_sp_nexthop_rif_fini(nh);
2727                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
2728                 break;
2729         }
2730 }
2731
2732 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp,
2733                                        struct mlxsw_sp_nexthop *nh,
2734                                        struct fib_nh *fib_nh)
2735 {
2736         struct mlxsw_sp_router *router = mlxsw_sp->router;
2737         struct net_device *dev = fib_nh->nh_dev;
2738         enum mlxsw_sp_ipip_type ipipt;
2739         struct mlxsw_sp_rif *rif;
2740         int err;
2741
2742         if (mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fib_nh, &ipipt) &&
2743             router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
2744                                                      MLXSW_SP_L3_PROTO_IPV4)) {
2745                 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
2746                 err = mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
2747                 if (err)
2748                         return err;
2749                 mlxsw_sp_nexthop_rif_init(nh, &nh->ipip_entry->ol_lb->common);
2750                 return 0;
2751         }
2752
2753         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
2754         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
2755         if (!rif)
2756                 return 0;
2757
2758         mlxsw_sp_nexthop_rif_init(nh, rif);
2759         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2760         if (err)
2761                 goto err_neigh_init;
2762
2763         return 0;
2764
2765 err_neigh_init:
2766         mlxsw_sp_nexthop_rif_fini(nh);
2767         return err;
2768 }
2769
2770 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp,
2771                                         struct mlxsw_sp_nexthop *nh)
2772 {
2773         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2774 }
2775
2776 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
2777                                   struct mlxsw_sp_nexthop_group *nh_grp,
2778                                   struct mlxsw_sp_nexthop *nh,
2779                                   struct fib_nh *fib_nh)
2780 {
2781         struct net_device *dev = fib_nh->nh_dev;
2782         struct in_device *in_dev;
2783         int err;
2784
2785         nh->nh_grp = nh_grp;
2786         nh->key.fib_nh = fib_nh;
2787         memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
2788         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
2789         if (err)
2790                 return err;
2791
2792         if (!dev)
2793                 return 0;
2794
2795         in_dev = __in_dev_get_rtnl(dev);
2796         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
2797             fib_nh->nh_flags & RTNH_F_LINKDOWN)
2798                 return 0;
2799
2800         err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2801         if (err)
2802                 goto err_nexthop_neigh_init;
2803
2804         return 0;
2805
2806 err_nexthop_neigh_init:
2807         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2808         return err;
2809 }
2810
2811 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
2812                                    struct mlxsw_sp_nexthop *nh)
2813 {
2814         mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2815         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2816 }
2817
2818 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
2819                                     unsigned long event, struct fib_nh *fib_nh)
2820 {
2821         struct mlxsw_sp_nexthop_key key;
2822         struct mlxsw_sp_nexthop *nh;
2823
2824         if (mlxsw_sp->router->aborted)
2825                 return;
2826
2827         key.fib_nh = fib_nh;
2828         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
2829         if (WARN_ON_ONCE(!nh))
2830                 return;
2831
2832         switch (event) {
2833         case FIB_EVENT_NH_ADD:
2834                 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2835                 break;
2836         case FIB_EVENT_NH_DEL:
2837                 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2838                 break;
2839         }
2840
2841         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2842 }
2843
2844 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2845                                            struct mlxsw_sp_rif *rif)
2846 {
2847         struct mlxsw_sp_nexthop *nh, *tmp;
2848
2849         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
2850                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2851                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2852         }
2853 }
2854
2855 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
2856                                    const struct fib_info *fi)
2857 {
2858         return fi->fib_nh->nh_scope == RT_SCOPE_LINK ||
2859                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL);
2860 }
2861
2862 static struct mlxsw_sp_nexthop_group *
2863 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
2864 {
2865         struct mlxsw_sp_nexthop_group *nh_grp;
2866         struct mlxsw_sp_nexthop *nh;
2867         struct fib_nh *fib_nh;
2868         size_t alloc_size;
2869         int i;
2870         int err;
2871
2872         alloc_size = sizeof(*nh_grp) +
2873                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
2874         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
2875         if (!nh_grp)
2876                 return ERR_PTR(-ENOMEM);
2877         nh_grp->priv = fi;
2878         INIT_LIST_HEAD(&nh_grp->fib_list);
2879         nh_grp->neigh_tbl = &arp_tbl;
2880
2881         nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
2882         nh_grp->count = fi->fib_nhs;
2883         fib_info_hold(fi);
2884         for (i = 0; i < nh_grp->count; i++) {
2885                 nh = &nh_grp->nexthops[i];
2886                 fib_nh = &fi->fib_nh[i];
2887                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
2888                 if (err)
2889                         goto err_nexthop4_init;
2890         }
2891         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
2892         if (err)
2893                 goto err_nexthop_group_insert;
2894         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2895         return nh_grp;
2896
2897 err_nexthop_group_insert:
2898 err_nexthop4_init:
2899         for (i--; i >= 0; i--) {
2900                 nh = &nh_grp->nexthops[i];
2901                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2902         }
2903         fib_info_put(fi);
2904         kfree(nh_grp);
2905         return ERR_PTR(err);
2906 }
2907
2908 static void
2909 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
2910                                 struct mlxsw_sp_nexthop_group *nh_grp)
2911 {
2912         struct mlxsw_sp_nexthop *nh;
2913         int i;
2914
2915         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
2916         for (i = 0; i < nh_grp->count; i++) {
2917                 nh = &nh_grp->nexthops[i];
2918                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2919         }
2920         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2921         WARN_ON_ONCE(nh_grp->adj_index_valid);
2922         fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
2923         kfree(nh_grp);
2924 }
2925
2926 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
2927                                        struct mlxsw_sp_fib_entry *fib_entry,
2928                                        struct fib_info *fi)
2929 {
2930         struct mlxsw_sp_nexthop_group *nh_grp;
2931
2932         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
2933         if (!nh_grp) {
2934                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
2935                 if (IS_ERR(nh_grp))
2936                         return PTR_ERR(nh_grp);
2937         }
2938         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
2939         fib_entry->nh_group = nh_grp;
2940         return 0;
2941 }
2942
2943 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
2944                                         struct mlxsw_sp_fib_entry *fib_entry)
2945 {
2946         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2947
2948         list_del(&fib_entry->nexthop_group_node);
2949         if (!list_empty(&nh_grp->fib_list))
2950                 return;
2951         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
2952 }
2953
2954 static bool
2955 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2956 {
2957         struct mlxsw_sp_fib4_entry *fib4_entry;
2958
2959         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
2960                                   common);
2961         return !fib4_entry->tos;
2962 }
2963
2964 static bool
2965 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2966 {
2967         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
2968
2969         switch (fib_entry->fib_node->fib->proto) {
2970         case MLXSW_SP_L3_PROTO_IPV4:
2971                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
2972                         return false;
2973                 break;
2974         case MLXSW_SP_L3_PROTO_IPV6:
2975                 break;
2976         }
2977
2978         switch (fib_entry->type) {
2979         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2980                 return !!nh_group->adj_index_valid;
2981         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2982                 return !!nh_group->nh_rif;
2983         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
2984                 return true;
2985         default:
2986                 return false;
2987         }
2988 }
2989
2990 static struct mlxsw_sp_nexthop *
2991 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
2992                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
2993 {
2994         int i;
2995
2996         for (i = 0; i < nh_grp->count; i++) {
2997                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2998                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2999
3000                 if (nh->rif && nh->rif->dev == rt->dst.dev &&
3001                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
3002                                     &rt->rt6i_gateway))
3003                         return nh;
3004                 continue;
3005         }
3006
3007         return NULL;
3008 }
3009
3010 static void
3011 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3012 {
3013         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3014         int i;
3015
3016         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL ||
3017             fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) {
3018                 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3019                 return;
3020         }
3021
3022         for (i = 0; i < nh_grp->count; i++) {
3023                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3024
3025                 if (nh->offloaded)
3026                         nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3027                 else
3028                         nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3029         }
3030 }
3031
3032 static void
3033 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3034 {
3035         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3036         int i;
3037
3038         for (i = 0; i < nh_grp->count; i++) {
3039                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3040
3041                 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3042         }
3043 }
3044
3045 static void
3046 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3047 {
3048         struct mlxsw_sp_fib6_entry *fib6_entry;
3049         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3050
3051         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3052                                   common);
3053
3054         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
3055                 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3056                                  list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
3057                 return;
3058         }
3059
3060         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3061                 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3062                 struct mlxsw_sp_nexthop *nh;
3063
3064                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3065                 if (nh && nh->offloaded)
3066                         mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
3067                 else
3068                         mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
3069         }
3070 }
3071
3072 static void
3073 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3074 {
3075         struct mlxsw_sp_fib6_entry *fib6_entry;
3076         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3077
3078         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3079                                   common);
3080         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3081                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3082
3083                 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
3084         }
3085 }
3086
3087 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3088 {
3089         switch (fib_entry->fib_node->fib->proto) {
3090         case MLXSW_SP_L3_PROTO_IPV4:
3091                 mlxsw_sp_fib4_entry_offload_set(fib_entry);
3092                 break;
3093         case MLXSW_SP_L3_PROTO_IPV6:
3094                 mlxsw_sp_fib6_entry_offload_set(fib_entry);
3095                 break;
3096         }
3097 }
3098
3099 static void
3100 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3101 {
3102         switch (fib_entry->fib_node->fib->proto) {
3103         case MLXSW_SP_L3_PROTO_IPV4:
3104                 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
3105                 break;
3106         case MLXSW_SP_L3_PROTO_IPV6:
3107                 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
3108                 break;
3109         }
3110 }
3111
3112 static void
3113 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
3114                                    enum mlxsw_reg_ralue_op op, int err)
3115 {
3116         switch (op) {
3117         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
3118                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
3119         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
3120                 if (err)
3121                         return;
3122                 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
3123                         mlxsw_sp_fib_entry_offload_set(fib_entry);
3124                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
3125                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
3126                 return;
3127         default:
3128                 return;
3129         }
3130 }
3131
3132 static void
3133 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
3134                               const struct mlxsw_sp_fib_entry *fib_entry,
3135                               enum mlxsw_reg_ralue_op op)
3136 {
3137         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
3138         enum mlxsw_reg_ralxx_protocol proto;
3139         u32 *p_dip;
3140
3141         proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
3142
3143         switch (fib->proto) {
3144         case MLXSW_SP_L3_PROTO_IPV4:
3145                 p_dip = (u32 *) fib_entry->fib_node->key.addr;
3146                 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
3147                                       fib_entry->fib_node->key.prefix_len,
3148                                       *p_dip);
3149                 break;
3150         case MLXSW_SP_L3_PROTO_IPV6:
3151                 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
3152                                       fib_entry->fib_node->key.prefix_len,
3153                                       fib_entry->fib_node->key.addr);
3154                 break;
3155         }
3156 }
3157
3158 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
3159                                         struct mlxsw_sp_fib_entry *fib_entry,
3160                                         enum mlxsw_reg_ralue_op op)
3161 {
3162         char ralue_pl[MLXSW_REG_RALUE_LEN];
3163         enum mlxsw_reg_ralue_trap_action trap_action;
3164         u16 trap_id = 0;
3165         u32 adjacency_index = 0;
3166         u16 ecmp_size = 0;
3167
3168         /* In case the nexthop group adjacency index is valid, use it
3169          * with provided ECMP size. Otherwise, setup trap and pass
3170          * traffic to kernel.
3171          */
3172         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
3173                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
3174                 adjacency_index = fib_entry->nh_group->adj_index;
3175                 ecmp_size = fib_entry->nh_group->ecmp_size;
3176         } else {
3177                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
3178                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
3179         }
3180
3181         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3182         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
3183                                         adjacency_index, ecmp_size);
3184         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3185 }
3186
3187 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
3188                                        struct mlxsw_sp_fib_entry *fib_entry,
3189                                        enum mlxsw_reg_ralue_op op)
3190 {
3191         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
3192         enum mlxsw_reg_ralue_trap_action trap_action;
3193         char ralue_pl[MLXSW_REG_RALUE_LEN];
3194         u16 trap_id = 0;
3195         u16 rif_index = 0;
3196
3197         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
3198                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
3199                 rif_index = rif->rif_index;
3200         } else {
3201                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
3202                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
3203         }
3204
3205         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3206         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
3207                                        rif_index);
3208         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3209 }
3210
3211 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
3212                                       struct mlxsw_sp_fib_entry *fib_entry,
3213                                       enum mlxsw_reg_ralue_op op)
3214 {
3215         char ralue_pl[MLXSW_REG_RALUE_LEN];
3216
3217         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3218         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
3219         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3220 }
3221
3222 static int
3223 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
3224                                  struct mlxsw_sp_fib_entry *fib_entry,
3225                                  enum mlxsw_reg_ralue_op op)
3226 {
3227         struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
3228         const struct mlxsw_sp_ipip_ops *ipip_ops;
3229
3230         if (WARN_ON(!ipip_entry))
3231                 return -EINVAL;
3232
3233         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3234         return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op,
3235                                       fib_entry->decap.tunnel_index);
3236 }
3237
3238 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3239                                    struct mlxsw_sp_fib_entry *fib_entry,
3240                                    enum mlxsw_reg_ralue_op op)
3241 {
3242         switch (fib_entry->type) {
3243         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
3244                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
3245         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
3246                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
3247         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
3248                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
3249         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
3250                 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
3251                                                         fib_entry, op);
3252         }
3253         return -EINVAL;
3254 }
3255
3256 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3257                                  struct mlxsw_sp_fib_entry *fib_entry,
3258                                  enum mlxsw_reg_ralue_op op)
3259 {
3260         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
3261
3262         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
3263
3264         return err;
3265 }
3266
3267 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
3268                                      struct mlxsw_sp_fib_entry *fib_entry)
3269 {
3270         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3271                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
3272 }
3273
3274 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
3275                                   struct mlxsw_sp_fib_entry *fib_entry)
3276 {
3277         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3278                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
3279 }
3280
3281 static int
3282 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
3283                              const struct fib_entry_notifier_info *fen_info,
3284                              struct mlxsw_sp_fib_entry *fib_entry)
3285 {
3286         union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
3287         struct net_device *dev = fen_info->fi->fib_dev;
3288         struct mlxsw_sp_ipip_entry *ipip_entry;
3289         struct fib_info *fi = fen_info->fi;
3290
3291         switch (fen_info->type) {
3292         case RTN_LOCAL:
3293                 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev,
3294                                                  MLXSW_SP_L3_PROTO_IPV4, dip);
3295                 if (ipip_entry) {
3296                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
3297                         return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
3298                                                              fib_entry,
3299                                                              ipip_entry);
3300                 }
3301                 /* fall through */
3302         case RTN_BROADCAST:
3303                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
3304                 return 0;
3305         case RTN_UNREACHABLE: /* fall through */
3306         case RTN_BLACKHOLE: /* fall through */
3307         case RTN_PROHIBIT:
3308                 /* Packets hitting these routes need to be trapped, but
3309                  * can do so with a lower priority than packets directed
3310                  * at the host, so use action type local instead of trap.
3311                  */
3312                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3313                 return 0;
3314         case RTN_UNICAST:
3315                 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
3316                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
3317                 else
3318                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3319                 return 0;
3320         default:
3321                 return -EINVAL;
3322         }
3323 }
3324
3325 static struct mlxsw_sp_fib4_entry *
3326 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
3327                            struct mlxsw_sp_fib_node *fib_node,
3328                            const struct fib_entry_notifier_info *fen_info)
3329 {
3330         struct mlxsw_sp_fib4_entry *fib4_entry;
3331         struct mlxsw_sp_fib_entry *fib_entry;
3332         int err;
3333
3334         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
3335         if (!fib4_entry)
3336                 return ERR_PTR(-ENOMEM);
3337         fib_entry = &fib4_entry->common;
3338
3339         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
3340         if (err)
3341                 goto err_fib4_entry_type_set;
3342
3343         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
3344         if (err)
3345                 goto err_nexthop4_group_get;
3346
3347         fib4_entry->prio = fen_info->fi->fib_priority;
3348         fib4_entry->tb_id = fen_info->tb_id;
3349         fib4_entry->type = fen_info->type;
3350         fib4_entry->tos = fen_info->tos;
3351
3352         fib_entry->fib_node = fib_node;
3353
3354         return fib4_entry;
3355
3356 err_nexthop4_group_get:
3357 err_fib4_entry_type_set:
3358         kfree(fib4_entry);
3359         return ERR_PTR(err);
3360 }
3361
3362 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
3363                                         struct mlxsw_sp_fib4_entry *fib4_entry)
3364 {
3365         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
3366         kfree(fib4_entry);
3367 }
3368
3369 static struct mlxsw_sp_fib4_entry *
3370 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
3371                            const struct fib_entry_notifier_info *fen_info)
3372 {
3373         struct mlxsw_sp_fib4_entry *fib4_entry;
3374         struct mlxsw_sp_fib_node *fib_node;
3375         struct mlxsw_sp_fib *fib;
3376         struct mlxsw_sp_vr *vr;
3377
3378         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
3379         if (!vr)
3380                 return NULL;
3381         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
3382
3383         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
3384                                             sizeof(fen_info->dst),
3385                                             fen_info->dst_len);
3386         if (!fib_node)
3387                 return NULL;
3388
3389         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3390                 if (fib4_entry->tb_id == fen_info->tb_id &&
3391                     fib4_entry->tos == fen_info->tos &&
3392                     fib4_entry->type == fen_info->type &&
3393                     mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
3394                     fen_info->fi) {
3395                         return fib4_entry;
3396                 }
3397         }
3398
3399         return NULL;
3400 }
3401
3402 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
3403         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
3404         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
3405         .key_len = sizeof(struct mlxsw_sp_fib_key),
3406         .automatic_shrinking = true,
3407 };
3408
3409 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
3410                                     struct mlxsw_sp_fib_node *fib_node)
3411 {
3412         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
3413                                       mlxsw_sp_fib_ht_params);
3414 }
3415
3416 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
3417                                      struct mlxsw_sp_fib_node *fib_node)
3418 {
3419         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
3420                                mlxsw_sp_fib_ht_params);
3421 }
3422
3423 static struct mlxsw_sp_fib_node *
3424 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
3425                          size_t addr_len, unsigned char prefix_len)
3426 {
3427         struct mlxsw_sp_fib_key key;
3428
3429         memset(&key, 0, sizeof(key));
3430         memcpy(key.addr, addr, addr_len);
3431         key.prefix_len = prefix_len;
3432         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
3433 }
3434
3435 static struct mlxsw_sp_fib_node *
3436 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
3437                          size_t addr_len, unsigned char prefix_len)
3438 {
3439         struct mlxsw_sp_fib_node *fib_node;
3440
3441         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
3442         if (!fib_node)
3443                 return NULL;
3444
3445         INIT_LIST_HEAD(&fib_node->entry_list);
3446         list_add(&fib_node->list, &fib->node_list);
3447         memcpy(fib_node->key.addr, addr, addr_len);
3448         fib_node->key.prefix_len = prefix_len;
3449
3450         return fib_node;
3451 }
3452
3453 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
3454 {
3455         list_del(&fib_node->list);
3456         WARN_ON(!list_empty(&fib_node->entry_list));
3457         kfree(fib_node);
3458 }
3459
3460 static bool
3461 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
3462                                  const struct mlxsw_sp_fib_entry *fib_entry)
3463 {
3464         return list_first_entry(&fib_node->entry_list,
3465                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
3466 }
3467
3468 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
3469                                       struct mlxsw_sp_fib *fib,
3470                                       struct mlxsw_sp_fib_node *fib_node)
3471 {
3472         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3473         struct mlxsw_sp_lpm_tree *lpm_tree;
3474         int err;
3475
3476         /* Since the tree is shared between all virtual routers we must
3477          * make sure it contains all the required prefix lengths. This
3478          * can be computed by either adding the new prefix length to the
3479          * existing prefix usage of a bound tree, or by aggregating the
3480          * prefix lengths across all virtual routers and adding the new
3481          * one as well.
3482          */
3483         if (fib->lpm_tree)
3484                 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
3485                                           &fib->lpm_tree->prefix_usage);
3486         else
3487                 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3488         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
3489
3490         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3491                                          fib->proto);
3492         if (IS_ERR(lpm_tree))
3493                 return PTR_ERR(lpm_tree);
3494
3495         if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
3496                 return 0;
3497
3498         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3499         if (err)
3500                 return err;
3501
3502         return 0;
3503 }
3504
3505 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
3506                                          struct mlxsw_sp_fib *fib)
3507 {
3508         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
3509                 return;
3510         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
3511         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
3512         fib->lpm_tree = NULL;
3513 }
3514
3515 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
3516 {
3517         unsigned char prefix_len = fib_node->key.prefix_len;
3518         struct mlxsw_sp_fib *fib = fib_node->fib;
3519
3520         if (fib->prefix_ref_count[prefix_len]++ == 0)
3521                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
3522 }
3523
3524 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
3525 {
3526         unsigned char prefix_len = fib_node->key.prefix_len;
3527         struct mlxsw_sp_fib *fib = fib_node->fib;
3528
3529         if (--fib->prefix_ref_count[prefix_len] == 0)
3530                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
3531 }
3532
3533 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
3534                                   struct mlxsw_sp_fib_node *fib_node,
3535                                   struct mlxsw_sp_fib *fib)
3536 {
3537         int err;
3538
3539         err = mlxsw_sp_fib_node_insert(fib, fib_node);
3540         if (err)
3541                 return err;
3542         fib_node->fib = fib;
3543
3544         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
3545         if (err)
3546                 goto err_fib_lpm_tree_link;
3547
3548         mlxsw_sp_fib_node_prefix_inc(fib_node);
3549
3550         return 0;
3551
3552 err_fib_lpm_tree_link:
3553         fib_node->fib = NULL;
3554         mlxsw_sp_fib_node_remove(fib, fib_node);
3555         return err;
3556 }
3557
3558 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
3559                                    struct mlxsw_sp_fib_node *fib_node)
3560 {
3561         struct mlxsw_sp_fib *fib = fib_node->fib;
3562
3563         mlxsw_sp_fib_node_prefix_dec(fib_node);
3564         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
3565         fib_node->fib = NULL;
3566         mlxsw_sp_fib_node_remove(fib, fib_node);
3567 }
3568
3569 static struct mlxsw_sp_fib_node *
3570 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
3571                       size_t addr_len, unsigned char prefix_len,
3572                       enum mlxsw_sp_l3proto proto)
3573 {
3574         struct mlxsw_sp_fib_node *fib_node;
3575         struct mlxsw_sp_fib *fib;
3576         struct mlxsw_sp_vr *vr;
3577         int err;
3578
3579         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
3580         if (IS_ERR(vr))
3581                 return ERR_CAST(vr);
3582         fib = mlxsw_sp_vr_fib(vr, proto);
3583
3584         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
3585         if (fib_node)
3586                 return fib_node;
3587
3588         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
3589         if (!fib_node) {
3590                 err = -ENOMEM;
3591                 goto err_fib_node_create;
3592         }
3593
3594         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
3595         if (err)
3596                 goto err_fib_node_init;
3597
3598         return fib_node;
3599
3600 err_fib_node_init:
3601         mlxsw_sp_fib_node_destroy(fib_node);
3602 err_fib_node_create:
3603         mlxsw_sp_vr_put(vr);
3604         return ERR_PTR(err);
3605 }
3606
3607 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
3608                                   struct mlxsw_sp_fib_node *fib_node)
3609 {
3610         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
3611
3612         if (!list_empty(&fib_node->entry_list))
3613                 return;
3614         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
3615         mlxsw_sp_fib_node_destroy(fib_node);
3616         mlxsw_sp_vr_put(vr);
3617 }
3618
3619 static struct mlxsw_sp_fib4_entry *
3620 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3621                               const struct mlxsw_sp_fib4_entry *new4_entry)
3622 {
3623         struct mlxsw_sp_fib4_entry *fib4_entry;
3624
3625         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3626                 if (fib4_entry->tb_id > new4_entry->tb_id)
3627                         continue;
3628                 if (fib4_entry->tb_id != new4_entry->tb_id)
3629                         break;
3630                 if (fib4_entry->tos > new4_entry->tos)
3631                         continue;
3632                 if (fib4_entry->prio >= new4_entry->prio ||
3633                     fib4_entry->tos < new4_entry->tos)
3634                         return fib4_entry;
3635         }
3636
3637         return NULL;
3638 }
3639
3640 static int
3641 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
3642                                struct mlxsw_sp_fib4_entry *new4_entry)
3643 {
3644         struct mlxsw_sp_fib_node *fib_node;
3645
3646         if (WARN_ON(!fib4_entry))
3647                 return -EINVAL;
3648
3649         fib_node = fib4_entry->common.fib_node;
3650         list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
3651                                  common.list) {
3652                 if (fib4_entry->tb_id != new4_entry->tb_id ||
3653                     fib4_entry->tos != new4_entry->tos ||
3654                     fib4_entry->prio != new4_entry->prio)
3655                         break;
3656         }
3657
3658         list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
3659         return 0;
3660 }
3661
3662 static int
3663 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
3664                                bool replace, bool append)
3665 {
3666         struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
3667         struct mlxsw_sp_fib4_entry *fib4_entry;
3668
3669         fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
3670
3671         if (append)
3672                 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
3673         if (replace && WARN_ON(!fib4_entry))
3674                 return -EINVAL;
3675
3676         /* Insert new entry before replaced one, so that we can later
3677          * remove the second.
3678          */
3679         if (fib4_entry) {
3680                 list_add_tail(&new4_entry->common.list,
3681                               &fib4_entry->common.list);
3682         } else {
3683                 struct mlxsw_sp_fib4_entry *last;
3684
3685                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3686                         if (new4_entry->tb_id > last->tb_id)
3687                                 break;
3688                         fib4_entry = last;
3689                 }
3690
3691                 if (fib4_entry)
3692                         list_add(&new4_entry->common.list,
3693                                  &fib4_entry->common.list);
3694                 else
3695                         list_add(&new4_entry->common.list,
3696                                  &fib_node->entry_list);
3697         }
3698
3699         return 0;
3700 }
3701
3702 static void
3703 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
3704 {
3705         list_del(&fib4_entry->common.list);
3706 }
3707
3708 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
3709                                        struct mlxsw_sp_fib_entry *fib_entry)
3710 {
3711         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3712
3713         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3714                 return 0;
3715
3716         /* To prevent packet loss, overwrite the previously offloaded
3717          * entry.
3718          */
3719         if (!list_is_singular(&fib_node->entry_list)) {
3720                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3721                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3722
3723                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
3724         }
3725
3726         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3727 }
3728
3729 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
3730                                         struct mlxsw_sp_fib_entry *fib_entry)
3731 {
3732         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3733
3734         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3735                 return;
3736
3737         /* Promote the next entry by overwriting the deleted entry */
3738         if (!list_is_singular(&fib_node->entry_list)) {
3739                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3740                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3741
3742                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
3743                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3744                 return;
3745         }
3746
3747         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
3748 }
3749
3750 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3751                                          struct mlxsw_sp_fib4_entry *fib4_entry,
3752                                          bool replace, bool append)
3753 {
3754         int err;
3755
3756         err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
3757         if (err)
3758                 return err;
3759
3760         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
3761         if (err)
3762                 goto err_fib_node_entry_add;
3763
3764         return 0;
3765
3766 err_fib_node_entry_add:
3767         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3768         return err;
3769 }
3770
3771 static void
3772 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3773                                 struct mlxsw_sp_fib4_entry *fib4_entry)
3774 {
3775         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
3776         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3777
3778         if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP)
3779                 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common);
3780 }
3781
3782 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
3783                                         struct mlxsw_sp_fib4_entry *fib4_entry,
3784                                         bool replace)
3785 {
3786         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
3787         struct mlxsw_sp_fib4_entry *replaced;
3788
3789         if (!replace)
3790                 return;
3791
3792         /* We inserted the new entry before replaced one */
3793         replaced = list_next_entry(fib4_entry, common.list);
3794
3795         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
3796         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
3797         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3798 }
3799
3800 static int
3801 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
3802                          const struct fib_entry_notifier_info *fen_info,
3803                          bool replace, bool append)
3804 {
3805         struct mlxsw_sp_fib4_entry *fib4_entry;
3806         struct mlxsw_sp_fib_node *fib_node;
3807         int err;
3808
3809         if (mlxsw_sp->router->aborted)
3810                 return 0;
3811
3812         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
3813                                          &fen_info->dst, sizeof(fen_info->dst),
3814                                          fen_info->dst_len,
3815                                          MLXSW_SP_L3_PROTO_IPV4);
3816         if (IS_ERR(fib_node)) {
3817                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
3818                 return PTR_ERR(fib_node);
3819         }
3820
3821         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
3822         if (IS_ERR(fib4_entry)) {
3823                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
3824                 err = PTR_ERR(fib4_entry);
3825                 goto err_fib4_entry_create;
3826         }
3827
3828         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
3829                                             append);
3830         if (err) {
3831                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
3832                 goto err_fib4_node_entry_link;
3833         }
3834
3835         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
3836
3837         return 0;
3838
3839 err_fib4_node_entry_link:
3840         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3841 err_fib4_entry_create:
3842         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3843         return err;
3844 }
3845
3846 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
3847                                      struct fib_entry_notifier_info *fen_info)
3848 {
3849         struct mlxsw_sp_fib4_entry *fib4_entry;
3850         struct mlxsw_sp_fib_node *fib_node;
3851
3852         if (mlxsw_sp->router->aborted)
3853                 return;
3854
3855         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
3856         if (WARN_ON(!fib4_entry))
3857                 return;
3858         fib_node = fib4_entry->common.fib_node;
3859
3860         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
3861         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3862         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3863 }
3864
3865 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
3866 {
3867         /* Packets with link-local destination IP arriving to the router
3868          * are trapped to the CPU, so no need to program specific routes
3869          * for them.
3870          */
3871         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
3872                 return true;
3873
3874         /* Multicast routes aren't supported, so ignore them. Neighbour
3875          * Discovery packets are specifically trapped.
3876          */
3877         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
3878                 return true;
3879
3880         /* Cloned routes are irrelevant in the forwarding path. */
3881         if (rt->rt6i_flags & RTF_CACHE)
3882                 return true;
3883
3884         return false;
3885 }
3886
3887 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
3888 {
3889         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3890
3891         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
3892         if (!mlxsw_sp_rt6)
3893                 return ERR_PTR(-ENOMEM);
3894
3895         /* In case of route replace, replaced route is deleted with
3896          * no notification. Take reference to prevent accessing freed
3897          * memory.
3898          */
3899         mlxsw_sp_rt6->rt = rt;
3900         rt6_hold(rt);
3901
3902         return mlxsw_sp_rt6;
3903 }
3904
3905 #if IS_ENABLED(CONFIG_IPV6)
3906 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3907 {
3908         rt6_release(rt);
3909 }
3910 #else
3911 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3912 {
3913 }
3914 #endif
3915
3916 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3917 {
3918         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
3919         kfree(mlxsw_sp_rt6);
3920 }
3921
3922 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
3923 {
3924         /* RTF_CACHE routes are ignored */
3925         return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
3926 }
3927
3928 static struct rt6_info *
3929 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
3930 {
3931         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3932                                 list)->rt;
3933 }
3934
3935 static struct mlxsw_sp_fib6_entry *
3936 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3937                                  const struct rt6_info *nrt, bool replace)
3938 {
3939         struct mlxsw_sp_fib6_entry *fib6_entry;
3940
3941         if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
3942                 return NULL;
3943
3944         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3945                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3946
3947                 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
3948                  * virtual router.
3949                  */
3950                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3951                         continue;
3952                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3953                         break;
3954                 if (rt->rt6i_metric < nrt->rt6i_metric)
3955                         continue;
3956                 if (rt->rt6i_metric == nrt->rt6i_metric &&
3957                     mlxsw_sp_fib6_rt_can_mp(rt))
3958                         return fib6_entry;
3959                 if (rt->rt6i_metric > nrt->rt6i_metric)
3960                         break;
3961         }
3962
3963         return NULL;
3964 }
3965
3966 static struct mlxsw_sp_rt6 *
3967 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
3968                             const struct rt6_info *rt)
3969 {
3970         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3971
3972         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3973                 if (mlxsw_sp_rt6->rt == rt)
3974                         return mlxsw_sp_rt6;
3975         }
3976
3977         return NULL;
3978 }
3979
3980 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3981                                         const struct rt6_info *rt,
3982                                         enum mlxsw_sp_ipip_type *ret)
3983 {
3984         return rt->dst.dev &&
3985                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->dst.dev, ret);
3986 }
3987
3988 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
3989                                        struct mlxsw_sp_nexthop_group *nh_grp,
3990                                        struct mlxsw_sp_nexthop *nh,
3991                                        const struct rt6_info *rt)
3992 {
3993         struct mlxsw_sp_router *router = mlxsw_sp->router;
3994         struct net_device *dev = rt->dst.dev;
3995         enum mlxsw_sp_ipip_type ipipt;
3996         struct mlxsw_sp_rif *rif;
3997         int err;
3998
3999         if (mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, &ipipt) &&
4000             router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
4001                                                      MLXSW_SP_L3_PROTO_IPV6)) {
4002                 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4003                 err = mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
4004                 if (err)
4005                         return err;
4006                 mlxsw_sp_nexthop_rif_init(nh, &nh->ipip_entry->ol_lb->common);
4007                 return 0;
4008         }
4009
4010         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4011         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4012         if (!rif)
4013                 return 0;
4014         mlxsw_sp_nexthop_rif_init(nh, rif);
4015
4016         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4017         if (err)
4018                 goto err_nexthop_neigh_init;
4019
4020         return 0;
4021
4022 err_nexthop_neigh_init:
4023         mlxsw_sp_nexthop_rif_fini(nh);
4024         return err;
4025 }
4026
4027 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
4028                                         struct mlxsw_sp_nexthop *nh)
4029 {
4030         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4031 }
4032
4033 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
4034                                   struct mlxsw_sp_nexthop_group *nh_grp,
4035                                   struct mlxsw_sp_nexthop *nh,
4036                                   const struct rt6_info *rt)
4037 {
4038         struct net_device *dev = rt->dst.dev;
4039
4040         nh->nh_grp = nh_grp;
4041         memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
4042
4043         if (!dev)
4044                 return 0;
4045         nh->ifindex = dev->ifindex;
4046
4047         return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt);
4048 }
4049
4050 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
4051                                    struct mlxsw_sp_nexthop *nh)
4052 {
4053         mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh);
4054 }
4055
4056 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4057                                     const struct rt6_info *rt)
4058 {
4059         return rt->rt6i_flags & RTF_GATEWAY ||
4060                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
4061 }
4062
4063 static struct mlxsw_sp_nexthop_group *
4064 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
4065                                struct mlxsw_sp_fib6_entry *fib6_entry)
4066 {
4067         struct mlxsw_sp_nexthop_group *nh_grp;
4068         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4069         struct mlxsw_sp_nexthop *nh;
4070         size_t alloc_size;
4071         int i = 0;
4072         int err;
4073
4074         alloc_size = sizeof(*nh_grp) +
4075                      fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
4076         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
4077         if (!nh_grp)
4078                 return ERR_PTR(-ENOMEM);
4079         INIT_LIST_HEAD(&nh_grp->fib_list);
4080 #if IS_ENABLED(CONFIG_IPV6)
4081         nh_grp->neigh_tbl = &nd_tbl;
4082 #endif
4083         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
4084                                         struct mlxsw_sp_rt6, list);
4085         nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
4086         nh_grp->count = fib6_entry->nrt6;
4087         for (i = 0; i < nh_grp->count; i++) {
4088                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
4089
4090                 nh = &nh_grp->nexthops[i];
4091                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
4092                 if (err)
4093                         goto err_nexthop6_init;
4094                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
4095         }
4096
4097         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4098         if (err)
4099                 goto err_nexthop_group_insert;
4100
4101         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4102         return nh_grp;
4103
4104 err_nexthop_group_insert:
4105 err_nexthop6_init:
4106         for (i--; i >= 0; i--) {
4107                 nh = &nh_grp->nexthops[i];
4108                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4109         }
4110         kfree(nh_grp);
4111         return ERR_PTR(err);
4112 }
4113
4114 static void
4115 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
4116                                 struct mlxsw_sp_nexthop_group *nh_grp)
4117 {
4118         struct mlxsw_sp_nexthop *nh;
4119         int i = nh_grp->count;
4120
4121         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4122         for (i--; i >= 0; i--) {
4123                 nh = &nh_grp->nexthops[i];
4124                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4125         }
4126         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4127         WARN_ON(nh_grp->adj_index_valid);
4128         kfree(nh_grp);
4129 }
4130
4131 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
4132                                        struct mlxsw_sp_fib6_entry *fib6_entry)
4133 {
4134         struct mlxsw_sp_nexthop_group *nh_grp;
4135
4136         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
4137         if (!nh_grp) {
4138                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
4139                 if (IS_ERR(nh_grp))
4140                         return PTR_ERR(nh_grp);
4141         }
4142
4143         list_add_tail(&fib6_entry->common.nexthop_group_node,
4144                       &nh_grp->fib_list);
4145         fib6_entry->common.nh_group = nh_grp;
4146
4147         return 0;
4148 }
4149
4150 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
4151                                         struct mlxsw_sp_fib_entry *fib_entry)
4152 {
4153         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4154
4155         list_del(&fib_entry->nexthop_group_node);
4156         if (!list_empty(&nh_grp->fib_list))
4157                 return;
4158         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
4159 }
4160
4161 static int
4162 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
4163                                struct mlxsw_sp_fib6_entry *fib6_entry)
4164 {
4165         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
4166         int err;
4167
4168         fib6_entry->common.nh_group = NULL;
4169         list_del(&fib6_entry->common.nexthop_group_node);
4170
4171         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4172         if (err)
4173                 goto err_nexthop6_group_get;
4174
4175         /* In case this entry is offloaded, then the adjacency index
4176          * currently associated with it in the device's table is that
4177          * of the old group. Start using the new one instead.
4178          */
4179         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4180         if (err)
4181                 goto err_fib_node_entry_add;
4182
4183         if (list_empty(&old_nh_grp->fib_list))
4184                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
4185
4186         return 0;
4187
4188 err_fib_node_entry_add:
4189         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4190 err_nexthop6_group_get:
4191         list_add_tail(&fib6_entry->common.nexthop_group_node,
4192                       &old_nh_grp->fib_list);
4193         fib6_entry->common.nh_group = old_nh_grp;
4194         return err;
4195 }
4196
4197 static int
4198 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
4199                                 struct mlxsw_sp_fib6_entry *fib6_entry,
4200                                 struct rt6_info *rt)
4201 {
4202         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4203         int err;
4204
4205         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4206         if (IS_ERR(mlxsw_sp_rt6))
4207                 return PTR_ERR(mlxsw_sp_rt6);
4208
4209         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4210         fib6_entry->nrt6++;
4211
4212         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4213         if (err)
4214                 goto err_nexthop6_group_update;
4215
4216         return 0;
4217
4218 err_nexthop6_group_update:
4219         fib6_entry->nrt6--;
4220         list_del(&mlxsw_sp_rt6->list);
4221         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4222         return err;
4223 }
4224
4225 static void
4226 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
4227                                 struct mlxsw_sp_fib6_entry *fib6_entry,
4228                                 struct rt6_info *rt)
4229 {
4230         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4231
4232         mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
4233         if (WARN_ON(!mlxsw_sp_rt6))
4234                 return;
4235
4236         fib6_entry->nrt6--;
4237         list_del(&mlxsw_sp_rt6->list);
4238         mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4239         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4240 }
4241
4242 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4243                                          struct mlxsw_sp_fib_entry *fib_entry,
4244                                          const struct rt6_info *rt)
4245 {
4246         /* Packets hitting RTF_REJECT routes need to be discarded by the
4247          * stack. We can rely on their destination device not having a
4248          * RIF (it's the loopback device) and can thus use action type
4249          * local, which will cause them to be trapped with a lower
4250          * priority than packets that need to be locally received.
4251          */
4252         if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
4253                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4254         else if (rt->rt6i_flags & RTF_REJECT)
4255                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4256         else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
4257                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4258         else
4259                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4260 }
4261
4262 static void
4263 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
4264 {
4265         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
4266
4267         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
4268                                  list) {
4269                 fib6_entry->nrt6--;
4270                 list_del(&mlxsw_sp_rt6->list);
4271                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4272         }
4273 }
4274
4275 static struct mlxsw_sp_fib6_entry *
4276 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
4277                            struct mlxsw_sp_fib_node *fib_node,
4278                            struct rt6_info *rt)
4279 {
4280         struct mlxsw_sp_fib6_entry *fib6_entry;
4281         struct mlxsw_sp_fib_entry *fib_entry;
4282         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4283         int err;
4284
4285         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
4286         if (!fib6_entry)
4287                 return ERR_PTR(-ENOMEM);
4288         fib_entry = &fib6_entry->common;
4289
4290         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4291         if (IS_ERR(mlxsw_sp_rt6)) {
4292                 err = PTR_ERR(mlxsw_sp_rt6);
4293                 goto err_rt6_create;
4294         }
4295
4296         mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt);
4297
4298         INIT_LIST_HEAD(&fib6_entry->rt6_list);
4299         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4300         fib6_entry->nrt6 = 1;
4301         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4302         if (err)
4303                 goto err_nexthop6_group_get;
4304
4305         fib_entry->fib_node = fib_node;
4306
4307         return fib6_entry;
4308
4309 err_nexthop6_group_get:
4310         list_del(&mlxsw_sp_rt6->list);
4311         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4312 err_rt6_create:
4313         kfree(fib6_entry);
4314         return ERR_PTR(err);
4315 }
4316
4317 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4318                                         struct mlxsw_sp_fib6_entry *fib6_entry)
4319 {
4320         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4321         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
4322         WARN_ON(fib6_entry->nrt6);
4323         kfree(fib6_entry);
4324 }
4325
4326 static struct mlxsw_sp_fib6_entry *
4327 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4328                               const struct rt6_info *nrt, bool replace)
4329 {
4330         struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
4331
4332         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4333                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4334
4335                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
4336                         continue;
4337                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
4338                         break;
4339                 if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
4340                         if (mlxsw_sp_fib6_rt_can_mp(rt) ==
4341                             mlxsw_sp_fib6_rt_can_mp(nrt))
4342                                 return fib6_entry;
4343                         if (mlxsw_sp_fib6_rt_can_mp(nrt))
4344                                 fallback = fallback ?: fib6_entry;
4345                 }
4346                 if (rt->rt6i_metric > nrt->rt6i_metric)
4347                         return fallback ?: fib6_entry;
4348         }
4349
4350         return fallback;
4351 }
4352
4353 static int
4354 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
4355                                bool replace)
4356 {
4357         struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
4358         struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
4359         struct mlxsw_sp_fib6_entry *fib6_entry;
4360
4361         fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
4362
4363         if (replace && WARN_ON(!fib6_entry))
4364                 return -EINVAL;
4365
4366         if (fib6_entry) {
4367                 list_add_tail(&new6_entry->common.list,
4368                               &fib6_entry->common.list);
4369         } else {
4370                 struct mlxsw_sp_fib6_entry *last;
4371
4372                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
4373                         struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
4374
4375                         if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
4376                                 break;
4377                         fib6_entry = last;
4378                 }
4379
4380                 if (fib6_entry)
4381                         list_add(&new6_entry->common.list,
4382                                  &fib6_entry->common.list);
4383                 else
4384                         list_add(&new6_entry->common.list,
4385                                  &fib_node->entry_list);
4386         }
4387
4388         return 0;
4389 }
4390
4391 static void
4392 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
4393 {
4394         list_del(&fib6_entry->common.list);
4395 }
4396
4397 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4398                                          struct mlxsw_sp_fib6_entry *fib6_entry,
4399                                          bool replace)
4400 {
4401         int err;
4402
4403         err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
4404         if (err)
4405                 return err;
4406
4407         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4408         if (err)
4409                 goto err_fib_node_entry_add;
4410
4411         return 0;
4412
4413 err_fib_node_entry_add:
4414         mlxsw_sp_fib6_node_list_remove(fib6_entry);
4415         return err;
4416 }
4417
4418 static void
4419 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4420                                 struct mlxsw_sp_fib6_entry *fib6_entry)
4421 {
4422         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
4423         mlxsw_sp_fib6_node_list_remove(fib6_entry);
4424 }
4425
4426 static struct mlxsw_sp_fib6_entry *
4427 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4428                            const struct rt6_info *rt)
4429 {
4430         struct mlxsw_sp_fib6_entry *fib6_entry;
4431         struct mlxsw_sp_fib_node *fib_node;
4432         struct mlxsw_sp_fib *fib;
4433         struct mlxsw_sp_vr *vr;
4434
4435         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
4436         if (!vr)
4437                 return NULL;
4438         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
4439
4440         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
4441                                             sizeof(rt->rt6i_dst.addr),
4442                                             rt->rt6i_dst.plen);
4443         if (!fib_node)
4444                 return NULL;
4445
4446         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4447                 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4448
4449                 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
4450                     rt->rt6i_metric == iter_rt->rt6i_metric &&
4451                     mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
4452                         return fib6_entry;
4453         }
4454
4455         return NULL;
4456 }
4457
4458 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
4459                                         struct mlxsw_sp_fib6_entry *fib6_entry,
4460                                         bool replace)
4461 {
4462         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
4463         struct mlxsw_sp_fib6_entry *replaced;
4464
4465         if (!replace)
4466                 return;
4467
4468         replaced = list_next_entry(fib6_entry, common.list);
4469
4470         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
4471         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
4472         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4473 }
4474
4475 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
4476                                     struct rt6_info *rt, bool replace)
4477 {
4478         struct mlxsw_sp_fib6_entry *fib6_entry;
4479         struct mlxsw_sp_fib_node *fib_node;
4480         int err;
4481
4482         if (mlxsw_sp->router->aborted)
4483                 return 0;
4484
4485         if (rt->rt6i_src.plen)
4486                 return -EINVAL;
4487
4488         if (mlxsw_sp_fib6_rt_should_ignore(rt))
4489                 return 0;
4490
4491         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
4492                                          &rt->rt6i_dst.addr,
4493                                          sizeof(rt->rt6i_dst.addr),
4494                                          rt->rt6i_dst.plen,
4495                                          MLXSW_SP_L3_PROTO_IPV6);
4496         if (IS_ERR(fib_node))
4497                 return PTR_ERR(fib_node);
4498
4499         /* Before creating a new entry, try to append route to an existing
4500          * multipath entry.
4501          */
4502         fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
4503         if (fib6_entry) {
4504                 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
4505                 if (err)
4506                         goto err_fib6_entry_nexthop_add;
4507                 return 0;
4508         }
4509
4510         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
4511         if (IS_ERR(fib6_entry)) {
4512                 err = PTR_ERR(fib6_entry);
4513                 goto err_fib6_entry_create;
4514         }
4515
4516         err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
4517         if (err)
4518                 goto err_fib6_node_entry_link;
4519
4520         mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
4521
4522         return 0;
4523
4524 err_fib6_node_entry_link:
4525         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4526 err_fib6_entry_create:
4527 err_fib6_entry_nexthop_add:
4528         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4529         return err;
4530 }
4531
4532 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
4533                                      struct rt6_info *rt)
4534 {
4535         struct mlxsw_sp_fib6_entry *fib6_entry;
4536         struct mlxsw_sp_fib_node *fib_node;
4537
4538         if (mlxsw_sp->router->aborted)
4539                 return;
4540
4541         if (mlxsw_sp_fib6_rt_should_ignore(rt))
4542                 return;
4543
4544         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
4545         if (WARN_ON(!fib6_entry))
4546                 return;
4547
4548         /* If route is part of a multipath entry, but not the last one
4549          * removed, then only reduce its nexthop group.
4550          */
4551         if (!list_is_singular(&fib6_entry->rt6_list)) {
4552                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
4553                 return;
4554         }
4555
4556         fib_node = fib6_entry->common.fib_node;
4557
4558         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4559         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4560         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4561 }
4562
4563 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
4564                                             enum mlxsw_reg_ralxx_protocol proto,
4565                                             u8 tree_id)
4566 {
4567         char ralta_pl[MLXSW_REG_RALTA_LEN];
4568         char ralst_pl[MLXSW_REG_RALST_LEN];
4569         int i, err;
4570
4571         mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
4572         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
4573         if (err)
4574                 return err;
4575
4576         mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
4577         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
4578         if (err)
4579                 return err;
4580
4581         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4582                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4583                 char raltb_pl[MLXSW_REG_RALTB_LEN];
4584                 char ralue_pl[MLXSW_REG_RALUE_LEN];
4585
4586                 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
4587                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
4588                                       raltb_pl);
4589                 if (err)
4590                         return err;
4591
4592                 mlxsw_reg_ralue_pack(ralue_pl, proto,
4593                                      MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
4594                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4595                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4596                                       ralue_pl);
4597                 if (err)
4598                         return err;
4599         }
4600
4601         return 0;
4602 }
4603
4604 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
4605 {
4606         enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
4607         int err;
4608
4609         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4610                                                MLXSW_SP_LPM_TREE_MIN);
4611         if (err)
4612                 return err;
4613
4614         proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
4615         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4616                                                 MLXSW_SP_LPM_TREE_MIN + 1);
4617 }
4618
4619 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
4620                                      struct mlxsw_sp_fib_node *fib_node)
4621 {
4622         struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
4623
4624         list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
4625                                  common.list) {
4626                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4627
4628                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4629                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4630                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4631                 /* Break when entry list is empty and node was freed.
4632                  * Otherwise, we'll access freed memory in the next
4633                  * iteration.
4634                  */
4635                 if (do_break)
4636                         break;
4637         }
4638 }
4639
4640 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
4641                                      struct mlxsw_sp_fib_node *fib_node)
4642 {
4643         struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
4644
4645         list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
4646                                  common.list) {
4647                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4648
4649                 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4650                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4651                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4652                 if (do_break)
4653                         break;
4654         }
4655 }
4656
4657 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
4658                                     struct mlxsw_sp_fib_node *fib_node)
4659 {
4660         switch (fib_node->fib->proto) {
4661         case MLXSW_SP_L3_PROTO_IPV4:
4662                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
4663                 break;
4664         case MLXSW_SP_L3_PROTO_IPV6:
4665                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
4666                 break;
4667         }
4668 }
4669
4670 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
4671                                   struct mlxsw_sp_vr *vr,
4672                                   enum mlxsw_sp_l3proto proto)
4673 {
4674         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
4675         struct mlxsw_sp_fib_node *fib_node, *tmp;
4676
4677         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
4678                 bool do_break = &tmp->list == &fib->node_list;
4679
4680                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
4681                 if (do_break)
4682                         break;
4683         }
4684 }
4685
4686 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
4687 {
4688         int i;
4689
4690         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4691                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4692
4693                 if (!mlxsw_sp_vr_is_used(vr))
4694                         continue;
4695                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
4696
4697                 /* If virtual router was only used for IPv4, then it's no
4698                  * longer used.
4699                  */
4700                 if (!mlxsw_sp_vr_is_used(vr))
4701                         continue;
4702                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
4703         }
4704 }
4705
4706 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
4707 {
4708         int err;
4709
4710         if (mlxsw_sp->router->aborted)
4711                 return;
4712         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
4713         mlxsw_sp_router_fib_flush(mlxsw_sp);
4714         mlxsw_sp->router->aborted = true;
4715         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
4716         if (err)
4717                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
4718 }
4719
4720 struct mlxsw_sp_fib_event_work {
4721         struct work_struct work;
4722         union {
4723                 struct fib6_entry_notifier_info fen6_info;
4724                 struct fib_entry_notifier_info fen_info;
4725                 struct fib_rule_notifier_info fr_info;
4726                 struct fib_nh_notifier_info fnh_info;
4727         };
4728         struct mlxsw_sp *mlxsw_sp;
4729         unsigned long event;
4730 };
4731
4732 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
4733 {
4734         struct mlxsw_sp_fib_event_work *fib_work =
4735                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4736         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4737         struct fib_rule *rule;
4738         bool replace, append;
4739         int err;
4740
4741         /* Protect internal structures from changes */
4742         rtnl_lock();
4743         switch (fib_work->event) {
4744         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4745         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4746         case FIB_EVENT_ENTRY_ADD:
4747                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4748                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
4749                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
4750                                                replace, append);
4751                 if (err)
4752                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4753                 fib_info_put(fib_work->fen_info.fi);
4754                 break;
4755         case FIB_EVENT_ENTRY_DEL:
4756                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
4757                 fib_info_put(fib_work->fen_info.fi);
4758                 break;
4759         case FIB_EVENT_RULE_ADD: /* fall through */
4760         case FIB_EVENT_RULE_DEL:
4761                 rule = fib_work->fr_info.rule;
4762                 if (!fib4_rule_default(rule) && !rule->l3mdev)
4763                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4764                 fib_rule_put(rule);
4765                 break;
4766         case FIB_EVENT_NH_ADD: /* fall through */
4767         case FIB_EVENT_NH_DEL:
4768                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
4769                                         fib_work->fnh_info.fib_nh);
4770                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
4771                 break;
4772         }
4773         rtnl_unlock();
4774         kfree(fib_work);
4775 }
4776
4777 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
4778 {
4779         struct mlxsw_sp_fib_event_work *fib_work =
4780                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4781         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4782         struct fib_rule *rule;
4783         bool replace;
4784         int err;
4785
4786         rtnl_lock();
4787         switch (fib_work->event) {
4788         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4789         case FIB_EVENT_ENTRY_ADD:
4790                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4791                 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
4792                                                fib_work->fen6_info.rt, replace);
4793                 if (err)
4794                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4795                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4796                 break;
4797         case FIB_EVENT_ENTRY_DEL:
4798                 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
4799                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4800                 break;
4801         case FIB_EVENT_RULE_ADD: /* fall through */
4802         case FIB_EVENT_RULE_DEL:
4803                 rule = fib_work->fr_info.rule;
4804                 if (!fib6_rule_default(rule) && !rule->l3mdev)
4805                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4806                 fib_rule_put(rule);
4807                 break;
4808         }
4809         rtnl_unlock();
4810         kfree(fib_work);
4811 }
4812
4813 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
4814                                        struct fib_notifier_info *info)
4815 {
4816         switch (fib_work->event) {
4817         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4818         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4819         case FIB_EVENT_ENTRY_ADD: /* fall through */
4820         case FIB_EVENT_ENTRY_DEL:
4821                 memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
4822                 /* Take referece on fib_info to prevent it from being
4823                  * freed while work is queued. Release it afterwards.
4824                  */
4825                 fib_info_hold(fib_work->fen_info.fi);
4826                 break;
4827         case FIB_EVENT_RULE_ADD: /* fall through */
4828         case FIB_EVENT_RULE_DEL:
4829                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4830                 fib_rule_get(fib_work->fr_info.rule);
4831                 break;
4832         case FIB_EVENT_NH_ADD: /* fall through */
4833         case FIB_EVENT_NH_DEL:
4834                 memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
4835                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
4836                 break;
4837         }
4838 }
4839
4840 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
4841                                        struct fib_notifier_info *info)
4842 {
4843         switch (fib_work->event) {
4844         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4845         case FIB_EVENT_ENTRY_ADD: /* fall through */
4846         case FIB_EVENT_ENTRY_DEL:
4847                 memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
4848                 rt6_hold(fib_work->fen6_info.rt);
4849                 break;
4850         case FIB_EVENT_RULE_ADD: /* fall through */
4851         case FIB_EVENT_RULE_DEL:
4852                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4853                 fib_rule_get(fib_work->fr_info.rule);
4854                 break;
4855         }
4856 }
4857
4858 /* Called with rcu_read_lock() */
4859 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
4860                                      unsigned long event, void *ptr)
4861 {
4862         struct mlxsw_sp_fib_event_work *fib_work;
4863         struct fib_notifier_info *info = ptr;
4864         struct mlxsw_sp_router *router;
4865
4866         if (!net_eq(info->net, &init_net) ||
4867             (info->family != AF_INET && info->family != AF_INET6))
4868                 return NOTIFY_DONE;
4869
4870         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
4871         if (WARN_ON(!fib_work))
4872                 return NOTIFY_BAD;
4873
4874         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
4875         fib_work->mlxsw_sp = router->mlxsw_sp;
4876         fib_work->event = event;
4877
4878         switch (info->family) {
4879         case AF_INET:
4880                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
4881                 mlxsw_sp_router_fib4_event(fib_work, info);
4882                 break;
4883         case AF_INET6:
4884                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
4885                 mlxsw_sp_router_fib6_event(fib_work, info);
4886                 break;
4887         }
4888
4889         mlxsw_core_schedule_work(&fib_work->work);
4890
4891         return NOTIFY_DONE;
4892 }
4893
4894 static struct mlxsw_sp_rif *
4895 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
4896                          const struct net_device *dev)
4897 {
4898         int i;
4899
4900         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
4901                 if (mlxsw_sp->router->rifs[i] &&
4902                     mlxsw_sp->router->rifs[i]->dev == dev)
4903                         return mlxsw_sp->router->rifs[i];
4904
4905         return NULL;
4906 }
4907
4908 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
4909 {
4910         char ritr_pl[MLXSW_REG_RITR_LEN];
4911         int err;
4912
4913         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
4914         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4915         if (WARN_ON_ONCE(err))
4916                 return err;
4917
4918         mlxsw_reg_ritr_enable_set(ritr_pl, false);
4919         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4920 }
4921
4922 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4923                                           struct mlxsw_sp_rif *rif)
4924 {
4925         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
4926         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
4927         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
4928 }
4929
4930 static bool
4931 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
4932                            unsigned long event)
4933 {
4934         struct inet6_dev *inet6_dev;
4935         bool addr_list_empty = true;
4936         struct in_device *idev;
4937
4938         switch (event) {
4939         case NETDEV_UP:
4940                 return rif == NULL;
4941         case NETDEV_DOWN:
4942                 idev = __in_dev_get_rtnl(dev);
4943                 if (idev && idev->ifa_list)
4944                         addr_list_empty = false;
4945
4946                 inet6_dev = __in6_dev_get(dev);
4947                 if (addr_list_empty && inet6_dev &&
4948                     !list_empty(&inet6_dev->addr_list))
4949                         addr_list_empty = false;
4950
4951                 if (rif && addr_list_empty &&
4952                     !netif_is_l3_slave(rif->dev))
4953                         return true;
4954                 /* It is possible we already removed the RIF ourselves
4955                  * if it was assigned to a netdev that is now a bridge
4956                  * or LAG slave.
4957                  */
4958                 return false;
4959         }
4960
4961         return false;
4962 }
4963
4964 static enum mlxsw_sp_rif_type
4965 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
4966                       const struct net_device *dev)
4967 {
4968         enum mlxsw_sp_fid_type type;
4969
4970         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
4971                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
4972
4973         /* Otherwise RIF type is derived from the type of the underlying FID. */
4974         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
4975                 type = MLXSW_SP_FID_TYPE_8021Q;
4976         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
4977                 type = MLXSW_SP_FID_TYPE_8021Q;
4978         else if (netif_is_bridge_master(dev))
4979                 type = MLXSW_SP_FID_TYPE_8021D;
4980         else
4981                 type = MLXSW_SP_FID_TYPE_RFID;
4982
4983         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
4984 }
4985
4986 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
4987 {
4988         int i;
4989
4990         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
4991                 if (!mlxsw_sp->router->rifs[i]) {
4992                         *p_rif_index = i;
4993                         return 0;
4994                 }
4995         }
4996
4997         return -ENOBUFS;
4998 }
4999
5000 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
5001                                                u16 vr_id,
5002                                                struct net_device *l3_dev)
5003 {
5004         struct mlxsw_sp_rif *rif;
5005
5006         rif = kzalloc(rif_size, GFP_KERNEL);
5007         if (!rif)
5008                 return NULL;
5009
5010         INIT_LIST_HEAD(&rif->nexthop_list);
5011         INIT_LIST_HEAD(&rif->neigh_list);
5012         ether_addr_copy(rif->addr, l3_dev->dev_addr);
5013         rif->mtu = l3_dev->mtu;
5014         rif->vr_id = vr_id;
5015         rif->dev = l3_dev;
5016         rif->rif_index = rif_index;
5017
5018         return rif;
5019 }
5020
5021 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
5022                                            u16 rif_index)
5023 {
5024         return mlxsw_sp->router->rifs[rif_index];
5025 }
5026
5027 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
5028 {
5029         return rif->rif_index;
5030 }
5031
5032 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
5033 {
5034         return lb_rif->common.rif_index;
5035 }
5036
5037 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
5038 {
5039         return lb_rif->ul_vr_id;
5040 }
5041
5042 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
5043 {
5044         return rif->dev->ifindex;
5045 }
5046
5047 static struct mlxsw_sp_rif *
5048 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
5049                     const struct mlxsw_sp_rif_params *params)
5050 {
5051         u32 tb_id = l3mdev_fib_table(params->dev);
5052         const struct mlxsw_sp_rif_ops *ops;
5053         struct mlxsw_sp_fid *fid = NULL;
5054         enum mlxsw_sp_rif_type type;
5055         struct mlxsw_sp_rif *rif;
5056         struct mlxsw_sp_vr *vr;
5057         u16 rif_index;
5058         int err;
5059
5060         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
5061         ops = mlxsw_sp->router->rif_ops_arr[type];
5062
5063         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
5064         if (IS_ERR(vr))
5065                 return ERR_CAST(vr);
5066         vr->rif_count++;
5067
5068         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
5069         if (err)
5070                 goto err_rif_index_alloc;
5071
5072         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
5073         if (!rif) {
5074                 err = -ENOMEM;
5075                 goto err_rif_alloc;
5076         }
5077         rif->mlxsw_sp = mlxsw_sp;
5078         rif->ops = ops;
5079
5080         if (ops->fid_get) {
5081                 fid = ops->fid_get(rif);
5082                 if (IS_ERR(fid)) {
5083                         err = PTR_ERR(fid);
5084                         goto err_fid_get;
5085                 }
5086                 rif->fid = fid;
5087         }
5088
5089         if (ops->setup)
5090                 ops->setup(rif, params);
5091
5092         err = ops->configure(rif);
5093         if (err)
5094                 goto err_configure;
5095
5096         mlxsw_sp_rif_counters_alloc(rif);
5097         mlxsw_sp->router->rifs[rif_index] = rif;
5098
5099         return rif;
5100
5101 err_configure:
5102         if (fid)
5103                 mlxsw_sp_fid_put(fid);
5104 err_fid_get:
5105         kfree(rif);
5106 err_rif_alloc:
5107 err_rif_index_alloc:
5108         vr->rif_count--;
5109         mlxsw_sp_vr_put(vr);
5110         return ERR_PTR(err);
5111 }
5112
5113 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
5114 {
5115         const struct mlxsw_sp_rif_ops *ops = rif->ops;
5116         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5117         struct mlxsw_sp_fid *fid = rif->fid;
5118         struct mlxsw_sp_vr *vr;
5119
5120         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
5121         vr = &mlxsw_sp->router->vrs[rif->vr_id];
5122
5123         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
5124         mlxsw_sp_rif_counters_free(rif);
5125         ops->deconfigure(rif);
5126         if (fid)
5127                 /* Loopback RIFs are not associated with a FID. */
5128                 mlxsw_sp_fid_put(fid);
5129         kfree(rif);
5130         vr->rif_count--;
5131         mlxsw_sp_vr_put(vr);
5132 }
5133
5134 static void
5135 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
5136                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
5137 {
5138         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5139
5140         params->vid = mlxsw_sp_port_vlan->vid;
5141         params->lag = mlxsw_sp_port->lagged;
5142         if (params->lag)
5143                 params->lag_id = mlxsw_sp_port->lag_id;
5144         else
5145                 params->system_port = mlxsw_sp_port->local_port;
5146 }
5147
5148 static int
5149 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
5150                                struct net_device *l3_dev)
5151 {
5152         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5153         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
5154         u16 vid = mlxsw_sp_port_vlan->vid;
5155         struct mlxsw_sp_rif *rif;
5156         struct mlxsw_sp_fid *fid;
5157         int err;
5158
5159         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5160         if (!rif) {
5161                 struct mlxsw_sp_rif_params params = {
5162                         .dev = l3_dev,
5163                 };
5164
5165                 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
5166                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
5167                 if (IS_ERR(rif))
5168                         return PTR_ERR(rif);
5169         }
5170
5171         /* FID was already created, just take a reference */
5172         fid = rif->ops->fid_get(rif);
5173         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
5174         if (err)
5175                 goto err_fid_port_vid_map;
5176
5177         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
5178         if (err)
5179                 goto err_port_vid_learning_set;
5180
5181         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
5182                                         BR_STATE_FORWARDING);
5183         if (err)
5184                 goto err_port_vid_stp_set;
5185
5186         mlxsw_sp_port_vlan->fid = fid;
5187
5188         return 0;
5189
5190 err_port_vid_stp_set:
5191         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
5192 err_port_vid_learning_set:
5193         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
5194 err_fid_port_vid_map:
5195         mlxsw_sp_fid_put(fid);
5196         return err;
5197 }
5198
5199 void
5200 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
5201 {
5202         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5203         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
5204         u16 vid = mlxsw_sp_port_vlan->vid;
5205
5206         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
5207                 return;
5208
5209         mlxsw_sp_port_vlan->fid = NULL;
5210         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
5211         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
5212         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
5213         /* If router port holds the last reference on the rFID, then the
5214          * associated Sub-port RIF will be destroyed.
5215          */
5216         mlxsw_sp_fid_put(fid);
5217 }
5218
5219 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
5220                                              struct net_device *port_dev,
5221                                              unsigned long event, u16 vid)
5222 {
5223         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
5224         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
5225
5226         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
5227         if (WARN_ON(!mlxsw_sp_port_vlan))
5228                 return -EINVAL;
5229
5230         switch (event) {
5231         case NETDEV_UP:
5232                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
5233                                                       l3_dev);
5234         case NETDEV_DOWN:
5235                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
5236                 break;
5237         }
5238
5239         return 0;
5240 }
5241
5242 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
5243                                         unsigned long event)
5244 {
5245         if (netif_is_bridge_port(port_dev) ||
5246             netif_is_lag_port(port_dev) ||
5247             netif_is_ovs_port(port_dev))
5248                 return 0;
5249
5250         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
5251 }
5252
5253 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
5254                                          struct net_device *lag_dev,
5255                                          unsigned long event, u16 vid)
5256 {
5257         struct net_device *port_dev;
5258         struct list_head *iter;
5259         int err;
5260
5261         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
5262                 if (mlxsw_sp_port_dev_check(port_dev)) {
5263                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
5264                                                                 port_dev,
5265                                                                 event, vid);
5266                         if (err)
5267                                 return err;
5268                 }
5269         }
5270
5271         return 0;
5272 }
5273
5274 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
5275                                        unsigned long event)
5276 {
5277         if (netif_is_bridge_port(lag_dev))
5278                 return 0;
5279
5280         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
5281 }
5282
5283 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
5284                                           unsigned long event)
5285 {
5286         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5287         struct mlxsw_sp_rif_params params = {
5288                 .dev = l3_dev,
5289         };
5290         struct mlxsw_sp_rif *rif;
5291
5292         switch (event) {
5293         case NETDEV_UP:
5294                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
5295                 if (IS_ERR(rif))
5296                         return PTR_ERR(rif);
5297                 break;
5298         case NETDEV_DOWN:
5299                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5300                 mlxsw_sp_rif_destroy(rif);
5301                 break;
5302         }
5303
5304         return 0;
5305 }
5306
5307 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
5308                                         unsigned long event)
5309 {
5310         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
5311         u16 vid = vlan_dev_vlan_id(vlan_dev);
5312
5313         if (netif_is_bridge_port(vlan_dev))
5314                 return 0;
5315
5316         if (mlxsw_sp_port_dev_check(real_dev))
5317                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
5318                                                          event, vid);
5319         else if (netif_is_lag_master(real_dev))
5320                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
5321                                                      vid);
5322         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
5323                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
5324
5325         return 0;
5326 }
5327
5328 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
5329                                      unsigned long event)
5330 {
5331         if (mlxsw_sp_port_dev_check(dev))
5332                 return mlxsw_sp_inetaddr_port_event(dev, event);
5333         else if (netif_is_lag_master(dev))
5334                 return mlxsw_sp_inetaddr_lag_event(dev, event);
5335         else if (netif_is_bridge_master(dev))
5336                 return mlxsw_sp_inetaddr_bridge_event(dev, event);
5337         else if (is_vlan_dev(dev))
5338                 return mlxsw_sp_inetaddr_vlan_event(dev, event);
5339         else
5340                 return 0;
5341 }
5342
5343 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
5344                             unsigned long event, void *ptr)
5345 {
5346         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
5347         struct net_device *dev = ifa->ifa_dev->dev;
5348         struct mlxsw_sp *mlxsw_sp;
5349         struct mlxsw_sp_rif *rif;
5350         int err = 0;
5351
5352         mlxsw_sp = mlxsw_sp_lower_get(dev);
5353         if (!mlxsw_sp)
5354                 goto out;
5355
5356         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5357         if (!mlxsw_sp_rif_should_config(rif, dev, event))
5358                 goto out;
5359
5360         err = __mlxsw_sp_inetaddr_event(dev, event);
5361 out:
5362         return notifier_from_errno(err);
5363 }
5364
5365 struct mlxsw_sp_inet6addr_event_work {
5366         struct work_struct work;
5367         struct net_device *dev;
5368         unsigned long event;
5369 };
5370
5371 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
5372 {
5373         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
5374                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
5375         struct net_device *dev = inet6addr_work->dev;
5376         unsigned long event = inet6addr_work->event;
5377         struct mlxsw_sp *mlxsw_sp;
5378         struct mlxsw_sp_rif *rif;
5379
5380         rtnl_lock();
5381         mlxsw_sp = mlxsw_sp_lower_get(dev);
5382         if (!mlxsw_sp)
5383                 goto out;
5384
5385         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5386         if (!mlxsw_sp_rif_should_config(rif, dev, event))
5387                 goto out;
5388
5389         __mlxsw_sp_inetaddr_event(dev, event);
5390 out:
5391         rtnl_unlock();
5392         dev_put(dev);
5393         kfree(inet6addr_work);
5394 }
5395
5396 /* Called with rcu_read_lock() */
5397 int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
5398                              unsigned long event, void *ptr)
5399 {
5400         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
5401         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
5402         struct net_device *dev = if6->idev->dev;
5403
5404         if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
5405                 return NOTIFY_DONE;
5406
5407         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
5408         if (!inet6addr_work)
5409                 return NOTIFY_BAD;
5410
5411         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
5412         inet6addr_work->dev = dev;
5413         inet6addr_work->event = event;
5414         dev_hold(dev);
5415         mlxsw_core_schedule_work(&inet6addr_work->work);
5416
5417         return NOTIFY_DONE;
5418 }
5419
5420 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
5421                              const char *mac, int mtu)
5422 {
5423         char ritr_pl[MLXSW_REG_RITR_LEN];
5424         int err;
5425
5426         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
5427         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5428         if (err)
5429                 return err;
5430
5431         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
5432         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
5433         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
5434         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5435 }
5436
5437 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
5438 {
5439         struct mlxsw_sp *mlxsw_sp;
5440         struct mlxsw_sp_rif *rif;
5441         u16 fid_index;
5442         int err;
5443
5444         mlxsw_sp = mlxsw_sp_lower_get(dev);
5445         if (!mlxsw_sp)
5446                 return 0;
5447
5448         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5449         if (!rif)
5450                 return 0;
5451         fid_index = mlxsw_sp_fid_index(rif->fid);
5452
5453         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
5454         if (err)
5455                 return err;
5456
5457         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
5458                                 dev->mtu);
5459         if (err)
5460                 goto err_rif_edit;
5461
5462         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
5463         if (err)
5464                 goto err_rif_fdb_op;
5465
5466         ether_addr_copy(rif->addr, dev->dev_addr);
5467         rif->mtu = dev->mtu;
5468
5469         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
5470
5471         return 0;
5472
5473 err_rif_fdb_op:
5474         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
5475 err_rif_edit:
5476         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
5477         return err;
5478 }
5479
5480 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
5481                                   struct net_device *l3_dev)
5482 {
5483         struct mlxsw_sp_rif *rif;
5484
5485         /* If netdev is already associated with a RIF, then we need to
5486          * destroy it and create a new one with the new virtual router ID.
5487          */
5488         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5489         if (rif)
5490                 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5491
5492         return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
5493 }
5494
5495 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
5496                                     struct net_device *l3_dev)
5497 {
5498         struct mlxsw_sp_rif *rif;
5499
5500         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5501         if (!rif)
5502                 return;
5503         __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5504 }
5505
5506 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
5507                                  struct netdev_notifier_changeupper_info *info)
5508 {
5509         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5510         int err = 0;
5511
5512         if (!mlxsw_sp)
5513                 return 0;
5514
5515         switch (event) {
5516         case NETDEV_PRECHANGEUPPER:
5517                 return 0;
5518         case NETDEV_CHANGEUPPER:
5519                 if (info->linking)
5520                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
5521                 else
5522                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
5523                 break;
5524         }
5525
5526         return err;
5527 }
5528
5529 static struct mlxsw_sp_rif_subport *
5530 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
5531 {
5532         return container_of(rif, struct mlxsw_sp_rif_subport, common);
5533 }
5534
5535 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
5536                                        const struct mlxsw_sp_rif_params *params)
5537 {
5538         struct mlxsw_sp_rif_subport *rif_subport;
5539
5540         rif_subport = mlxsw_sp_rif_subport_rif(rif);
5541         rif_subport->vid = params->vid;
5542         rif_subport->lag = params->lag;
5543         if (params->lag)
5544                 rif_subport->lag_id = params->lag_id;
5545         else
5546                 rif_subport->system_port = params->system_port;
5547 }
5548
5549 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
5550 {
5551         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5552         struct mlxsw_sp_rif_subport *rif_subport;
5553         char ritr_pl[MLXSW_REG_RITR_LEN];
5554
5555         rif_subport = mlxsw_sp_rif_subport_rif(rif);
5556         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
5557                             rif->rif_index, rif->vr_id, rif->dev->mtu);
5558         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5559         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
5560                                   rif_subport->lag ? rif_subport->lag_id :
5561                                                      rif_subport->system_port,
5562                                   rif_subport->vid);
5563
5564         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5565 }
5566
5567 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
5568 {
5569         int err;
5570
5571         err = mlxsw_sp_rif_subport_op(rif, true);
5572         if (err)
5573                 return err;
5574
5575         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5576                                   mlxsw_sp_fid_index(rif->fid), true);
5577         if (err)
5578                 goto err_rif_fdb_op;
5579
5580         mlxsw_sp_fid_rif_set(rif->fid, rif);
5581         return 0;
5582
5583 err_rif_fdb_op:
5584         mlxsw_sp_rif_subport_op(rif, false);
5585         return err;
5586 }
5587
5588 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
5589 {
5590         struct mlxsw_sp_fid *fid = rif->fid;
5591
5592         mlxsw_sp_fid_rif_set(fid, NULL);
5593         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5594                             mlxsw_sp_fid_index(fid), false);
5595         mlxsw_sp_rif_subport_op(rif, false);
5596 }
5597
5598 static struct mlxsw_sp_fid *
5599 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
5600 {
5601         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
5602 }
5603
5604 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
5605         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
5606         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
5607         .setup                  = mlxsw_sp_rif_subport_setup,
5608         .configure              = mlxsw_sp_rif_subport_configure,
5609         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
5610         .fid_get                = mlxsw_sp_rif_subport_fid_get,
5611 };
5612
5613 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
5614                                     enum mlxsw_reg_ritr_if_type type,
5615                                     u16 vid_fid, bool enable)
5616 {
5617         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5618         char ritr_pl[MLXSW_REG_RITR_LEN];
5619
5620         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
5621                             rif->dev->mtu);
5622         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5623         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
5624
5625         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5626 }
5627
5628 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
5629 {
5630         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
5631 }
5632
5633 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
5634 {
5635         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5636         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5637         int err;
5638
5639         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
5640         if (err)
5641                 return err;
5642
5643         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5644                                      mlxsw_sp_router_port(mlxsw_sp), true);
5645         if (err)
5646                 goto err_fid_mc_flood_set;
5647
5648         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5649                                      mlxsw_sp_router_port(mlxsw_sp), true);
5650         if (err)
5651                 goto err_fid_bc_flood_set;
5652
5653         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5654                                   mlxsw_sp_fid_index(rif->fid), true);
5655         if (err)
5656                 goto err_rif_fdb_op;
5657
5658         mlxsw_sp_fid_rif_set(rif->fid, rif);
5659         return 0;
5660
5661 err_rif_fdb_op:
5662         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5663                                mlxsw_sp_router_port(mlxsw_sp), false);
5664 err_fid_bc_flood_set:
5665         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5666                                mlxsw_sp_router_port(mlxsw_sp), false);
5667 err_fid_mc_flood_set:
5668         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5669         return err;
5670 }
5671
5672 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
5673 {
5674         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5675         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5676         struct mlxsw_sp_fid *fid = rif->fid;
5677
5678         mlxsw_sp_fid_rif_set(fid, NULL);
5679         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5680                             mlxsw_sp_fid_index(fid), false);
5681         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5682                                mlxsw_sp_router_port(mlxsw_sp), false);
5683         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5684                                mlxsw_sp_router_port(mlxsw_sp), false);
5685         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5686 }
5687
5688 static struct mlxsw_sp_fid *
5689 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
5690 {
5691         u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
5692
5693         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
5694 }
5695
5696 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
5697         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
5698         .rif_size               = sizeof(struct mlxsw_sp_rif),
5699         .configure              = mlxsw_sp_rif_vlan_configure,
5700         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
5701         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
5702 };
5703
5704 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
5705 {
5706         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5707         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5708         int err;
5709
5710         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
5711                                        true);
5712         if (err)
5713                 return err;
5714
5715         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5716                                      mlxsw_sp_router_port(mlxsw_sp), true);
5717         if (err)
5718                 goto err_fid_mc_flood_set;
5719
5720         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5721                                      mlxsw_sp_router_port(mlxsw_sp), true);
5722         if (err)
5723                 goto err_fid_bc_flood_set;
5724
5725         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5726                                   mlxsw_sp_fid_index(rif->fid), true);
5727         if (err)
5728                 goto err_rif_fdb_op;
5729
5730         mlxsw_sp_fid_rif_set(rif->fid, rif);
5731         return 0;
5732
5733 err_rif_fdb_op:
5734         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5735                                mlxsw_sp_router_port(mlxsw_sp), false);
5736 err_fid_bc_flood_set:
5737         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5738                                mlxsw_sp_router_port(mlxsw_sp), false);
5739 err_fid_mc_flood_set:
5740         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5741         return err;
5742 }
5743
5744 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
5745 {
5746         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5747         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5748         struct mlxsw_sp_fid *fid = rif->fid;
5749
5750         mlxsw_sp_fid_rif_set(fid, NULL);
5751         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5752                             mlxsw_sp_fid_index(fid), false);
5753         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5754                                mlxsw_sp_router_port(mlxsw_sp), false);
5755         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5756                                mlxsw_sp_router_port(mlxsw_sp), false);
5757         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5758 }
5759
5760 static struct mlxsw_sp_fid *
5761 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
5762 {
5763         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
5764 }
5765
5766 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
5767         .type                   = MLXSW_SP_RIF_TYPE_FID,
5768         .rif_size               = sizeof(struct mlxsw_sp_rif),
5769         .configure              = mlxsw_sp_rif_fid_configure,
5770         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
5771         .fid_get                = mlxsw_sp_rif_fid_fid_get,
5772 };
5773
5774 static struct mlxsw_sp_rif_ipip_lb *
5775 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
5776 {
5777         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
5778 }
5779
5780 static void
5781 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
5782                            const struct mlxsw_sp_rif_params *params)
5783 {
5784         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
5785         struct mlxsw_sp_rif_ipip_lb *rif_lb;
5786
5787         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
5788                                  common);
5789         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
5790         rif_lb->lb_config = params_lb->lb_config;
5791 }
5792
5793 static int
5794 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
5795                         struct mlxsw_sp_vr *ul_vr, bool enable)
5796 {
5797         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
5798         struct mlxsw_sp_rif *rif = &lb_rif->common;
5799         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5800         char ritr_pl[MLXSW_REG_RITR_LEN];
5801         u32 saddr4;
5802
5803         switch (lb_cf.ul_protocol) {
5804         case MLXSW_SP_L3_PROTO_IPV4:
5805                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
5806                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
5807                                     rif->rif_index, rif->vr_id, rif->dev->mtu);
5808                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
5809                             MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
5810                             ul_vr->id, saddr4, lb_cf.okey);
5811                 break;
5812
5813         case MLXSW_SP_L3_PROTO_IPV6:
5814                 return -EAFNOSUPPORT;
5815         }
5816
5817         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5818 }
5819
5820 static int
5821 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
5822 {
5823         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5824         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
5825         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5826         struct mlxsw_sp_vr *ul_vr;
5827         int err;
5828
5829         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id);
5830         if (IS_ERR(ul_vr))
5831                 return PTR_ERR(ul_vr);
5832
5833         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
5834         if (err)
5835                 goto err_loopback_op;
5836
5837         lb_rif->ul_vr_id = ul_vr->id;
5838         ++ul_vr->rif_count;
5839         return 0;
5840
5841 err_loopback_op:
5842         mlxsw_sp_vr_put(ul_vr);
5843         return err;
5844 }
5845
5846 static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
5847 {
5848         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5849         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5850         struct mlxsw_sp_vr *ul_vr;
5851
5852         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
5853         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
5854
5855         --ul_vr->rif_count;
5856         mlxsw_sp_vr_put(ul_vr);
5857 }
5858
5859 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
5860         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
5861         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
5862         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
5863         .configure              = mlxsw_sp_rif_ipip_lb_configure,
5864         .deconfigure            = mlxsw_sp_rif_ipip_lb_deconfigure,
5865 };
5866
5867 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
5868         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
5869         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_ops,
5870         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
5871         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp_rif_ipip_lb_ops,
5872 };
5873
5874 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
5875 {
5876         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5877
5878         mlxsw_sp->router->rifs = kcalloc(max_rifs,
5879                                          sizeof(struct mlxsw_sp_rif *),
5880                                          GFP_KERNEL);
5881         if (!mlxsw_sp->router->rifs)
5882                 return -ENOMEM;
5883
5884         mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
5885
5886         return 0;
5887 }
5888
5889 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
5890 {
5891         int i;
5892
5893         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5894                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
5895
5896         kfree(mlxsw_sp->router->rifs);
5897 }
5898
5899 static int
5900 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
5901 {
5902         char tigcr_pl[MLXSW_REG_TIGCR_LEN];
5903
5904         mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
5905         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
5906 }
5907
5908 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
5909 {
5910         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
5911         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
5912         return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
5913 }
5914
5915 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
5916 {
5917         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
5918 }
5919
5920 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
5921 {
5922         struct mlxsw_sp_router *router;
5923
5924         /* Flush pending FIB notifications and then flush the device's
5925          * table before requesting another dump. The FIB notification
5926          * block is unregistered, so no need to take RTNL.
5927          */
5928         mlxsw_core_flush_owq();
5929         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
5930         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
5931 }
5932
5933 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5934 {
5935         char rgcr_pl[MLXSW_REG_RGCR_LEN];
5936         u64 max_rifs;
5937         int err;
5938
5939         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
5940                 return -EIO;
5941         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5942
5943         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
5944         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
5945         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5946         if (err)
5947                 return err;
5948         return 0;
5949 }
5950
5951 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5952 {
5953         char rgcr_pl[MLXSW_REG_RGCR_LEN];
5954
5955         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
5956         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5957 }
5958
5959 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5960 {
5961         struct mlxsw_sp_router *router;
5962         int err;
5963
5964         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
5965         if (!router)
5966                 return -ENOMEM;
5967         mlxsw_sp->router = router;
5968         router->mlxsw_sp = mlxsw_sp;
5969
5970         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
5971         err = __mlxsw_sp_router_init(mlxsw_sp);
5972         if (err)
5973                 goto err_router_init;
5974
5975         err = mlxsw_sp_rifs_init(mlxsw_sp);
5976         if (err)
5977                 goto err_rifs_init;
5978
5979         err = mlxsw_sp_ipips_init(mlxsw_sp);
5980         if (err)
5981                 goto err_ipips_init;
5982
5983         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
5984                               &mlxsw_sp_nexthop_ht_params);
5985         if (err)
5986                 goto err_nexthop_ht_init;
5987
5988         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
5989                               &mlxsw_sp_nexthop_group_ht_params);
5990         if (err)
5991                 goto err_nexthop_group_ht_init;
5992
5993         err = mlxsw_sp_lpm_init(mlxsw_sp);
5994         if (err)
5995                 goto err_lpm_init;
5996
5997         err = mlxsw_sp_vrs_init(mlxsw_sp);
5998         if (err)
5999                 goto err_vrs_init;
6000
6001         err = mlxsw_sp_neigh_init(mlxsw_sp);
6002         if (err)
6003                 goto err_neigh_init;
6004
6005         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
6006         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
6007                                     mlxsw_sp_router_fib_dump_flush);
6008         if (err)
6009                 goto err_register_fib_notifier;
6010
6011         return 0;
6012
6013 err_register_fib_notifier:
6014         mlxsw_sp_neigh_fini(mlxsw_sp);
6015 err_neigh_init:
6016         mlxsw_sp_vrs_fini(mlxsw_sp);
6017 err_vrs_init:
6018         mlxsw_sp_lpm_fini(mlxsw_sp);
6019 err_lpm_init:
6020         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
6021 err_nexthop_group_ht_init:
6022         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
6023 err_nexthop_ht_init:
6024         mlxsw_sp_ipips_fini(mlxsw_sp);
6025 err_ipips_init:
6026         mlxsw_sp_rifs_fini(mlxsw_sp);
6027 err_rifs_init:
6028         __mlxsw_sp_router_fini(mlxsw_sp);
6029 err_router_init:
6030         kfree(mlxsw_sp->router);
6031         return err;
6032 }
6033
6034 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
6035 {
6036         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
6037         mlxsw_sp_neigh_fini(mlxsw_sp);
6038         mlxsw_sp_vrs_fini(mlxsw_sp);
6039         mlxsw_sp_lpm_fini(mlxsw_sp);
6040         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
6041         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
6042         mlxsw_sp_ipips_fini(mlxsw_sp);
6043         mlxsw_sp_rifs_fini(mlxsw_sp);
6044         __mlxsw_sp_router_fini(mlxsw_sp);
6045         kfree(mlxsw_sp->router);
6046 }