drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  * 3. Neither the names of the copyright holders nor the names of its
  18  *    contributors may be used to endorse or promote products derived from
  19  *    this software without specific prior written permission.
  20  *
  21  * Alternatively, this software may be distributed under the terms of the
  22  * GNU General Public License ("GPL") version 2 as published by the Free
  23  * Software Foundation.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  26  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  29  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  35  * POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 #include <linux/kernel.h>
  39 #include <linux/types.h>
  40 #include <linux/rhashtable.h>
  41 #include <linux/bitops.h>
  42 #include <linux/in6.h>
  43 #include <linux/notifier.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/netdevice.h>
  46 #include <linux/if_bridge.h>
  47 #include <linux/socket.h>
  48 #include <linux/route.h>
  49 #include <net/netevent.h>
  50 #include <net/neighbour.h>
  51 #include <net/arp.h>
  52 #include <net/ip_fib.h>
  53 #include <net/ip6_fib.h>
  54 #include <net/fib_rules.h>
  55 #include <net/ip_tunnels.h>
  56 #include <net/l3mdev.h>
  57 #include <net/addrconf.h>
  58 #include <net/ndisc.h>
  59 #include <net/ipv6.h>
  60 #include <net/fib_notifier.h>
  61
  62 #include "spectrum.h"
  63 #include "core.h"
  64 #include "reg.h"
  65 #include "spectrum_cnt.h"
  66 #include "spectrum_dpipe.h"
  67 #include "spectrum_ipip.h"
  68 #include "spectrum_router.h"
  69
  70 struct mlxsw_sp_vr;
  71 struct mlxsw_sp_lpm_tree;
  72 struct mlxsw_sp_rif_ops;
  73
  74 struct mlxsw_sp_router {
  75         struct mlxsw_sp *mlxsw_sp;
  76         struct mlxsw_sp_rif **rifs;
  77         struct mlxsw_sp_vr *vrs;
  78         struct rhashtable neigh_ht;
  79         struct rhashtable nexthop_group_ht;
  80         struct rhashtable nexthop_ht;
  81         struct {
  82                 struct mlxsw_sp_lpm_tree *trees;
  83                 unsigned int tree_count;
  84         } lpm;
  85         struct {
  86                 struct delayed_work dw;
  87                 unsigned long interval; /* ms */
  88         } neighs_update;
  89         struct delayed_work nexthop_probe_dw;
  90 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  91         struct list_head nexthop_neighs_list;
  92         struct list_head ipip_list;
  93         bool aborted;
  94         struct notifier_block fib_nb;
  95         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  96         const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
  97 };
  98
  99 struct mlxsw_sp_rif {
 100         struct list_head nexthop_list;
 101         struct list_head neigh_list;
 102         struct net_device *dev;
 103         struct mlxsw_sp_fid *fid;
 104         unsigned char addr[ETH_ALEN];
 105         int mtu;
 106         u16 rif_index;
 107         u16 vr_id;
 108         const struct mlxsw_sp_rif_ops *ops;
 109         struct mlxsw_sp *mlxsw_sp;
 110
 111         unsigned int counter_ingress;
 112         bool counter_ingress_valid;
 113         unsigned int counter_egress;
 114         bool counter_egress_valid;
 115 };
 116
 117 struct mlxsw_sp_rif_params {
 118         struct net_device *dev;
 119         union {
 120                 u16 system_port;
 121                 u16 lag_id;
 122         };
 123         u16 vid;
 124         bool lag;
 125 };
 126
 127 struct mlxsw_sp_rif_subport {
 128         struct mlxsw_sp_rif common;
 129         union {
 130                 u16 system_port;
 131                 u16 lag_id;
 132         };
 133         u16 vid;
 134         bool lag;
 135 };
 136
 137 struct mlxsw_sp_rif_ipip_lb {
 138         struct mlxsw_sp_rif common;
 139         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 140         u16 ul_vr_id; /* Reserved for Spectrum-2. */
 141 };
 142
 143 struct mlxsw_sp_rif_params_ipip_lb {
 144         struct mlxsw_sp_rif_params common;
 145         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 146 };
 147
 148 struct mlxsw_sp_rif_ops {
 149         enum mlxsw_sp_rif_type type;
 150         size_t rif_size;
 151
 152         void (*setup)(struct mlxsw_sp_rif *rif,
 153                       const struct mlxsw_sp_rif_params *params);
 154         int (*configure)(struct mlxsw_sp_rif *rif);
 155         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 156         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 157 };
 158
 159 static unsigned int *
 160 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 161                            enum mlxsw_sp_rif_counter_dir dir)
 162 {
 163         switch (dir) {
 164         case MLXSW_SP_RIF_COUNTER_EGRESS:
 165                 return &rif->counter_egress;
 166         case MLXSW_SP_RIF_COUNTER_INGRESS:
 167                 return &rif->counter_ingress;
 168         }
 169         return NULL;
 170 }
 171
 172 static bool
 173 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 174                                enum mlxsw_sp_rif_counter_dir dir)
 175 {
 176         switch (dir) {
 177         case MLXSW_SP_RIF_COUNTER_EGRESS:
 178                 return rif->counter_egress_valid;
 179         case MLXSW_SP_RIF_COUNTER_INGRESS:
 180                 return rif->counter_ingress_valid;
 181         }
 182         return false;
 183 }
 184
 185 static void
 186 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 187                                enum mlxsw_sp_rif_counter_dir dir,
 188                                bool valid)
 189 {
 190         switch (dir) {
 191         case MLXSW_SP_RIF_COUNTER_EGRESS:
 192                 rif->counter_egress_valid = valid;
 193                 break;
 194         case MLXSW_SP_RIF_COUNTER_INGRESS:
 195                 rif->counter_ingress_valid = valid;
 196                 break;
 197         }
 198 }
 199
 200 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 201                                      unsigned int counter_index, bool enable,
 202                                      enum mlxsw_sp_rif_counter_dir dir)
 203 {
 204         char ritr_pl[MLXSW_REG_RITR_LEN];
 205         bool is_egress = false;
 206         int err;
 207
 208         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 209                 is_egress = true;
 210         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 211         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 212         if (err)
 213                 return err;
 214
 215         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 216                                     is_egress);
 217         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 218 }
 219
 220 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 221                                    struct mlxsw_sp_rif *rif,
 222                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 223 {
 224         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 225         unsigned int *p_counter_index;
 226         bool valid;
 227         int err;
 228
 229         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 230         if (!valid)
 231                 return -EINVAL;
 232
 233         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 234         if (!p_counter_index)
 235                 return -EINVAL;
 236         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 237                              MLXSW_REG_RICNT_OPCODE_NOP);
 238         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 239         if (err)
 240                 return err;
 241         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 242         return 0;
 243 }
 244
 245 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 246                                       unsigned int counter_index)
 247 {
 248         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 249
 250         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 251                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 252         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 253 }
 254
 255 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 256                                struct mlxsw_sp_rif *rif,
 257                                enum mlxsw_sp_rif_counter_dir dir)
 258 {
 259         unsigned int *p_counter_index;
 260         int err;
 261
 262         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 263         if (!p_counter_index)
 264                 return -EINVAL;
 265         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 266                                      p_counter_index);
 267         if (err)
 268                 return err;
 269
 270         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 271         if (err)
 272                 goto err_counter_clear;
 273
 274         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 275                                         *p_counter_index, true, dir);
 276         if (err)
 277                 goto err_counter_edit;
 278         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 279         return 0;
 280
 281 err_counter_edit:
 282 err_counter_clear:
 283         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 284                               *p_counter_index);
 285         return err;
 286 }
 287
 288 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 289                                struct mlxsw_sp_rif *rif,
 290                                enum mlxsw_sp_rif_counter_dir dir)
 291 {
 292         unsigned int *p_counter_index;
 293
 294         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 295                 return;
 296
 297         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 298         if (WARN_ON(!p_counter_index))
 299                 return;
 300         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 301                                   *p_counter_index, false, dir);
 302         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 303                               *p_counter_index);
 304         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 305 }
 306
 307 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 308 {
 309         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 310         struct devlink *devlink;
 311
 312         devlink = priv_to_devlink(mlxsw_sp->core);
 313         if (!devlink_dpipe_table_counter_enabled(devlink,
 314                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 315                 return;
 316         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 317 }
 318
 319 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 320 {
 321         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 322
 323         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 324 }
 325
 326 static struct mlxsw_sp_rif *
 327 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 328                          const struct net_device *dev);
 329
 330 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 331
 332 struct mlxsw_sp_prefix_usage {
 333         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 334 };
 335
 336 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 337         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 338
 339 static bool
 340 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 341                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 342 {
 343         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 344 }
 345
 346 static bool
 347 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
 348 {
 349         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
 350
 351         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 352 }
 353
 354 static void
 355 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 356                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 357 {
 358         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 359 }
 360
 361 static void
 362 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 363                           unsigned char prefix_len)
 364 {
 365         set_bit(prefix_len, prefix_usage->b);
 366 }
 367
 368 static void
 369 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 370                             unsigned char prefix_len)
 371 {
 372         clear_bit(prefix_len, prefix_usage->b);
 373 }
 374
 375 struct mlxsw_sp_fib_key {
 376         unsigned char addr[sizeof(struct in6_addr)];
 377         unsigned char prefix_len;
 378 };
 379
 380 enum mlxsw_sp_fib_entry_type {
 381         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 382         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 383         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 384
 385         /* This is a special case of local delivery, where a packet should be
 386          * decapsulated on reception. Note that there is no corresponding ENCAP,
 387          * because that's a type of next hop, not of FIB entry. (There can be
 388          * several next hops in a REMOTE entry, and some of them may be
 389          * encapsulating entries.)
 390          */
 391         MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 392 };
 393
 394 struct mlxsw_sp_nexthop_group;
 395 struct mlxsw_sp_fib;
 396
 397 struct mlxsw_sp_fib_node {
 398         struct list_head entry_list;
 399         struct list_head list;
 400         struct rhash_head ht_node;
 401         struct mlxsw_sp_fib *fib;
 402         struct mlxsw_sp_fib_key key;
 403 };
 404
 405 struct mlxsw_sp_fib_entry_decap {
 406         struct mlxsw_sp_ipip_entry *ipip_entry;
 407         u32 tunnel_index;
 408 };
 409
 410 struct mlxsw_sp_fib_entry {
 411         struct list_head list;
 412         struct mlxsw_sp_fib_node *fib_node;
 413         enum mlxsw_sp_fib_entry_type type;
 414         struct list_head nexthop_group_node;
 415         struct mlxsw_sp_nexthop_group *nh_group;
 416         struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 417 };
 418
 419 struct mlxsw_sp_fib4_entry {
 420         struct mlxsw_sp_fib_entry common;
 421         u32 tb_id;
 422         u32 prio;
 423         u8 tos;
 424         u8 type;
 425 };
 426
 427 struct mlxsw_sp_fib6_entry {
 428         struct mlxsw_sp_fib_entry common;
 429         struct list_head rt6_list;
 430         unsigned int nrt6;
 431 };
 432
 433 struct mlxsw_sp_rt6 {
 434         struct list_head list;
 435         struct rt6_info *rt;
 436 };
 437
 438 struct mlxsw_sp_lpm_tree {
 439         u8 id; /* tree ID */
 440         unsigned int ref_count;
 441         enum mlxsw_sp_l3proto proto;
 442         struct mlxsw_sp_prefix_usage prefix_usage;
 443 };
 444
 445 struct mlxsw_sp_fib {
 446         struct rhashtable ht;
 447         struct list_head node_list;
 448         struct mlxsw_sp_vr *vr;
 449         struct mlxsw_sp_lpm_tree *lpm_tree;
 450         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 451         struct mlxsw_sp_prefix_usage prefix_usage;
 452         enum mlxsw_sp_l3proto proto;
 453 };
 454
 455 struct mlxsw_sp_vr {
 456         u16 id; /* virtual router ID */
 457         u32 tb_id; /* kernel fib table id */
 458         unsigned int rif_count;
 459         struct mlxsw_sp_fib *fib4;
 460         struct mlxsw_sp_fib *fib6;
 461 };
 462
 463 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 464
 465 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 466                                                 enum mlxsw_sp_l3proto proto)
 467 {
 468         struct mlxsw_sp_fib *fib;
 469         int err;
 470
 471         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 472         if (!fib)
 473                 return ERR_PTR(-ENOMEM);
 474         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 475         if (err)
 476                 goto err_rhashtable_init;
 477         INIT_LIST_HEAD(&fib->node_list);
 478         fib->proto = proto;
 479         fib->vr = vr;
 480         return fib;
 481
 482 err_rhashtable_init:
 483         kfree(fib);
 484         return ERR_PTR(err);
 485 }
 486
 487 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 488 {
 489         WARN_ON(!list_empty(&fib->node_list));
 490         WARN_ON(fib->lpm_tree);
 491         rhashtable_destroy(&fib->ht);
 492         kfree(fib);
 493 }
 494
 495 static struct mlxsw_sp_lpm_tree *
 496 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 497 {
 498         static struct mlxsw_sp_lpm_tree *lpm_tree;
 499         int i;
 500
 501         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 502                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 503                 if (lpm_tree->ref_count == 0)
 504                         return lpm_tree;
 505         }
 506         return NULL;
 507 }
 508
 509 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 510                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 511 {
 512         char ralta_pl[MLXSW_REG_RALTA_LEN];
 513
 514         mlxsw_reg_ralta_pack(ralta_pl, true,
 515                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 516                              lpm_tree->id);
 517         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 518 }
 519
 520 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 521                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 522 {
 523         char ralta_pl[MLXSW_REG_RALTA_LEN];
 524
 525         mlxsw_reg_ralta_pack(ralta_pl, false,
 526                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 527                              lpm_tree->id);
 528         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 529 }
 530
 531 static int
 532 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 533                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 534                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 535 {
 536         char ralst_pl[MLXSW_REG_RALST_LEN];
 537         u8 root_bin = 0;
 538         u8 prefix;
 539         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 540
 541         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 542                 root_bin = prefix;
 543
 544         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 545         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 546                 if (prefix == 0)
 547                         continue;
 548                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 549                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 550                 last_prefix = prefix;
 551         }
 552         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 553 }
 554
 555 static struct mlxsw_sp_lpm_tree *
 556 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 557                          struct mlxsw_sp_prefix_usage *prefix_usage,
 558                          enum mlxsw_sp_l3proto proto)
 559 {
 560         struct mlxsw_sp_lpm_tree *lpm_tree;
 561         int err;
 562
 563         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 564         if (!lpm_tree)
 565                 return ERR_PTR(-EBUSY);
 566         lpm_tree->proto = proto;
 567         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 568         if (err)
 569                 return ERR_PTR(err);
 570
 571         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 572                                                 lpm_tree);
 573         if (err)
 574                 goto err_left_struct_set;
 575         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 576                sizeof(lpm_tree->prefix_usage));
 577         return lpm_tree;
 578
 579 err_left_struct_set:
 580         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 581         return ERR_PTR(err);
 582 }
 583
 584 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 585                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 586 {
 587         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 588 }
 589
 590 static struct mlxsw_sp_lpm_tree *
 591 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 592                       struct mlxsw_sp_prefix_usage *prefix_usage,
 593                       enum mlxsw_sp_l3proto proto)
 594 {
 595         struct mlxsw_sp_lpm_tree *lpm_tree;
 596         int i;
 597
 598         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 599                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 600                 if (lpm_tree->ref_count != 0 &&
 601                     lpm_tree->proto == proto &&
 602                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 603                                              prefix_usage))
 604                         return lpm_tree;
 605         }
 606         return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
 607 }
 608
 609 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 610 {
 611         lpm_tree->ref_count++;
 612 }
 613
 614 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 615                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 616 {
 617         if (--lpm_tree->ref_count == 0)
 618                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 619 }
 620
 621 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 622
 623 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 624 {
 625         struct mlxsw_sp_lpm_tree *lpm_tree;
 626         u64 max_trees;
 627         int i;
 628
 629         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 630                 return -EIO;
 631
 632         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 633         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 634         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 635                                              sizeof(struct mlxsw_sp_lpm_tree),
 636                                              GFP_KERNEL);
 637         if (!mlxsw_sp->router->lpm.trees)
 638                 return -ENOMEM;
 639
 640         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 641                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 642                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 643         }
 644
 645         return 0;
 646 }
 647
 648 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 649 {
 650         kfree(mlxsw_sp->router->lpm.trees);
 651 }
 652
 653 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 654 {
 655         return !!vr->fib4 || !!vr->fib6;
 656 }
 657
 658 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 659 {
 660         struct mlxsw_sp_vr *vr;
 661         int i;
 662
 663         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 664                 vr = &mlxsw_sp->router->vrs[i];
 665                 if (!mlxsw_sp_vr_is_used(vr))
 666                         return vr;
 667         }
 668         return NULL;
 669 }
 670
 671 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 672                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 673 {
 674         char raltb_pl[MLXSW_REG_RALTB_LEN];
 675
 676         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 677                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 678                              tree_id);
 679         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 680 }
 681
 682 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 683                                        const struct mlxsw_sp_fib *fib)
 684 {
 685         char raltb_pl[MLXSW_REG_RALTB_LEN];
 686
 687         /* Bind to tree 0 which is default */
 688         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 689                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 690         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 691 }
 692
 693 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 694 {
 695         /* For our purpose, squash main and local table into one */
 696         if (tb_id == RT_TABLE_LOCAL)
 697                 tb_id = RT_TABLE_MAIN;
 698         return tb_id;
 699 }
 700
 701 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 702                                             u32 tb_id)
 703 {
 704         struct mlxsw_sp_vr *vr;
 705         int i;
 706
 707         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 708
 709         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 710                 vr = &mlxsw_sp->router->vrs[i];
 711                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 712                         return vr;
 713         }
 714         return NULL;
 715 }
 716
 717 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 718                                             enum mlxsw_sp_l3proto proto)
 719 {
 720         switch (proto) {
 721         case MLXSW_SP_L3_PROTO_IPV4:
 722                 return vr->fib4;
 723         case MLXSW_SP_L3_PROTO_IPV6:
 724                 return vr->fib6;
 725         }
 726         return NULL;
 727 }
 728
 729 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 730                                               u32 tb_id)
 731 {
 732         struct mlxsw_sp_vr *vr;
 733         int err;
 734
 735         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 736         if (!vr)
 737                 return ERR_PTR(-EBUSY);
 738         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 739         if (IS_ERR(vr->fib4))
 740                 return ERR_CAST(vr->fib4);
 741         vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
 742         if (IS_ERR(vr->fib6)) {
 743                 err = PTR_ERR(vr->fib6);
 744                 goto err_fib6_create;
 745         }
 746         vr->tb_id = tb_id;
 747         return vr;
 748
 749 err_fib6_create:
 750         mlxsw_sp_fib_destroy(vr->fib4);
 751         vr->fib4 = NULL;
 752         return ERR_PTR(err);
 753 }
 754
 755 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 756 {
 757         mlxsw_sp_fib_destroy(vr->fib6);
 758         vr->fib6 = NULL;
 759         mlxsw_sp_fib_destroy(vr->fib4);
 760         vr->fib4 = NULL;
 761 }
 762
 763 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 764 {
 765         struct mlxsw_sp_vr *vr;
 766
 767         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 768         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 769         if (!vr)
 770                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 771         return vr;
 772 }
 773
 774 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 775 {
 776         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 777             list_empty(&vr->fib6->node_list))
 778                 mlxsw_sp_vr_destroy(vr);
 779 }
 780
 781 static bool
 782 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 783                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 784 {
 785         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 786
 787         if (!mlxsw_sp_vr_is_used(vr))
 788                 return false;
 789         if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
 790                 return true;
 791         return false;
 792 }
 793
 794 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 795                                         struct mlxsw_sp_fib *fib,
 796                                         struct mlxsw_sp_lpm_tree *new_tree)
 797 {
 798         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 799         int err;
 800
 801         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 802         if (err)
 803                 return err;
 804         fib->lpm_tree = new_tree;
 805         mlxsw_sp_lpm_tree_hold(new_tree);
 806         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 807         return 0;
 808 }
 809
 810 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 811                                          struct mlxsw_sp_fib *fib,
 812                                          struct mlxsw_sp_lpm_tree *new_tree)
 813 {
 814         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 815         enum mlxsw_sp_l3proto proto = fib->proto;
 816         u8 old_id, new_id = new_tree->id;
 817         struct mlxsw_sp_vr *vr;
 818         int i, err;
 819
 820         if (!old_tree)
 821                 goto no_replace;
 822         old_id = old_tree->id;
 823
 824         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 825                 vr = &mlxsw_sp->router->vrs[i];
 826                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 827                         continue;
 828                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 829                                                    mlxsw_sp_vr_fib(vr, proto),
 830                                                    new_tree);
 831                 if (err)
 832                         goto err_tree_replace;
 833         }
 834
 835         return 0;
 836
 837 err_tree_replace:
 838         for (i--; i >= 0; i--) {
 839                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 840                         continue;
 841                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 842                                              mlxsw_sp_vr_fib(vr, proto),
 843                                              old_tree);
 844         }
 845         return err;
 846
 847 no_replace:
 848         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 849         if (err)
 850                 return err;
 851         fib->lpm_tree = new_tree;
 852         mlxsw_sp_lpm_tree_hold(new_tree);
 853         return 0;
 854 }
 855
 856 static void
 857 mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
 858                       enum mlxsw_sp_l3proto proto,
 859                       struct mlxsw_sp_prefix_usage *req_prefix_usage)
 860 {
 861         int i;
 862
 863         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 864                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
 865                 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 866                 unsigned char prefix;
 867
 868                 if (!mlxsw_sp_vr_is_used(vr))
 869                         continue;
 870                 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
 871                         mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
 872         }
 873 }
 874
 875 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 876 {
 877         struct mlxsw_sp_vr *vr;
 878         u64 max_vrs;
 879         int i;
 880
 881         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 882                 return -EIO;
 883
 884         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 885         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 886                                         GFP_KERNEL);
 887         if (!mlxsw_sp->router->vrs)
 888                 return -ENOMEM;
 889
 890         for (i = 0; i < max_vrs; i++) {
 891                 vr = &mlxsw_sp->router->vrs[i];
 892                 vr->id = i;
 893         }
 894
 895         return 0;
 896 }
 897
 898 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 899
 900 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 901 {
 902         /* At this stage we're guaranteed not to have new incoming
 903          * FIB notifications and the work queue is free from FIBs
 904          * sitting on top of mlxsw netdevs. However, we can still
 905          * have other FIBs queued. Flush the queue before flushing
 906          * the device's tables. No need for locks, as we're the only
 907          * writer.
 908          */
 909         mlxsw_core_flush_owq();
 910         mlxsw_sp_router_fib_flush(mlxsw_sp);
 911         kfree(mlxsw_sp->router->vrs);
 912 }
 913
 914 static struct net_device *
 915 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
 916 {
 917         struct ip_tunnel *tun = netdev_priv(ol_dev);
 918         struct net *net = dev_net(ol_dev);
 919
 920         return __dev_get_by_index(net, tun->parms.link);
 921 }
 922
 923 static u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
 924 {
 925         struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
 926
 927         if (d)
 928                 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
 929         else
 930                 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
 931 }
 932
 933 static struct mlxsw_sp_rif *
 934 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 935                     const struct mlxsw_sp_rif_params *params);
 936
 937 static struct mlxsw_sp_rif_ipip_lb *
 938 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
 939                                 enum mlxsw_sp_ipip_type ipipt,
 940                                 struct net_device *ol_dev)
 941 {
 942         struct mlxsw_sp_rif_params_ipip_lb lb_params;
 943         const struct mlxsw_sp_ipip_ops *ipip_ops;
 944         struct mlxsw_sp_rif *rif;
 945
 946         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
 947         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
 948                 .common.dev = ol_dev,
 949                 .common.lag = false,
 950                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
 951         };
 952
 953         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common);
 954         if (IS_ERR(rif))
 955                 return ERR_CAST(rif);
 956         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
 957 }
 958
 959 static struct mlxsw_sp_ipip_entry *
 960 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
 961                           enum mlxsw_sp_ipip_type ipipt,
 962                           struct net_device *ol_dev)
 963 {
 964         struct mlxsw_sp_ipip_entry *ipip_entry;
 965         struct mlxsw_sp_ipip_entry *ret = NULL;
 966
 967         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
 968         if (!ipip_entry)
 969                 return ERR_PTR(-ENOMEM);
 970
 971         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
 972                                                             ol_dev);
 973         if (IS_ERR(ipip_entry->ol_lb)) {
 974                 ret = ERR_CAST(ipip_entry->ol_lb);
 975                 goto err_ol_ipip_lb_create;
 976         }
 977
 978         ipip_entry->ipipt = ipipt;
 979         ipip_entry->ol_dev = ol_dev;
 980
 981         return ipip_entry;
 982
 983 err_ol_ipip_lb_create:
 984         kfree(ipip_entry);
 985         return ret;
 986 }
 987
 988 static void
 989 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp_ipip_entry *ipip_entry)
 990 {
 991         WARN_ON(ipip_entry->ref_count > 0);
 992         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
 993         kfree(ipip_entry);
 994 }
 995
 996 static __be32
 997 mlxsw_sp_ipip_netdev_saddr4(const struct net_device *ol_dev)
 998 {
 999         struct ip_tunnel *tun = netdev_priv(ol_dev);
1000
1001         return tun->parms.iph.saddr;
1002 }
1003
1004 union mlxsw_sp_l3addr
1005 mlxsw_sp_ipip_netdev_saddr(enum mlxsw_sp_l3proto proto,
1006                            const struct net_device *ol_dev)
1007 {
1008         switch (proto) {
1009         case MLXSW_SP_L3_PROTO_IPV4:
1010                 return (union mlxsw_sp_l3addr) {
1011                         .addr4 = mlxsw_sp_ipip_netdev_saddr4(ol_dev),
1012                 };
1013         case MLXSW_SP_L3_PROTO_IPV6:
1014                 break;
1015         };
1016
1017         WARN_ON(1);
1018         return (union mlxsw_sp_l3addr) {
1019                 .addr4 = 0,
1020         };
1021 }
1022
1023 __be32 mlxsw_sp_ipip_netdev_daddr4(const struct net_device *ol_dev)
1024 {
1025         struct ip_tunnel *tun = netdev_priv(ol_dev);
1026
1027         return tun->parms.iph.daddr;
1028 }
1029
1030 union mlxsw_sp_l3addr
1031 mlxsw_sp_ipip_netdev_daddr(enum mlxsw_sp_l3proto proto,
1032                            const struct net_device *ol_dev)
1033 {
1034         switch (proto) {
1035         case MLXSW_SP_L3_PROTO_IPV4:
1036                 return (union mlxsw_sp_l3addr) {
1037                         .addr4 = mlxsw_sp_ipip_netdev_daddr4(ol_dev),
1038                 };
1039         case MLXSW_SP_L3_PROTO_IPV6:
1040                 break;
1041         };
1042
1043         WARN_ON(1);
1044         return (union mlxsw_sp_l3addr) {
1045                 .addr4 = 0,
1046         };
1047 }
1048
1049 static bool mlxsw_sp_l3addr_eq(const union mlxsw_sp_l3addr *addr1,
1050                                const union mlxsw_sp_l3addr *addr2)
1051 {
1052         return !memcmp(addr1, addr2, sizeof(*addr1));
1053 }
1054
1055 static bool
1056 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1057                                   const enum mlxsw_sp_l3proto ul_proto,
1058                                   union mlxsw_sp_l3addr saddr,
1059                                   u32 ul_tb_id,
1060                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1061 {
1062         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1063         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1064         union mlxsw_sp_l3addr tun_saddr;
1065
1066         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1067                 return false;
1068
1069         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1070         return tun_ul_tb_id == ul_tb_id &&
1071                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1072 }
1073
1074 static int
1075 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1076                               struct mlxsw_sp_fib_entry *fib_entry,
1077                               struct mlxsw_sp_ipip_entry *ipip_entry)
1078 {
1079         u32 tunnel_index;
1080         int err;
1081
1082         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, 1, &tunnel_index);
1083         if (err)
1084                 return err;
1085
1086         ipip_entry->decap_fib_entry = fib_entry;
1087         fib_entry->decap.ipip_entry = ipip_entry;
1088         fib_entry->decap.tunnel_index = tunnel_index;
1089         return 0;
1090 }
1091
1092 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1093                                           struct mlxsw_sp_fib_entry *fib_entry)
1094 {
1095         /* Unlink this node from the IPIP entry that it's the decap entry of. */
1096         fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1097         fib_entry->decap.ipip_entry = NULL;
1098         mlxsw_sp_kvdl_free(mlxsw_sp, fib_entry->decap.tunnel_index);
1099 }
1100
1101 static struct mlxsw_sp_fib_node *
1102 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1103                          size_t addr_len, unsigned char prefix_len);
1104 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1105                                      struct mlxsw_sp_fib_entry *fib_entry);
1106
1107 static void
1108 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1109                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1110 {
1111         struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1112
1113         mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1114         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1115
1116         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1117 }
1118
1119 static void
1120 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1121                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1122                                   struct mlxsw_sp_fib_entry *decap_fib_entry)
1123 {
1124         if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1125                                           ipip_entry))
1126                 return;
1127         decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1128
1129         if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1130                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1131 }
1132
1133 /* Given an IPIP entry, find the corresponding decap route. */
1134 static struct mlxsw_sp_fib_entry *
1135 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1136                                struct mlxsw_sp_ipip_entry *ipip_entry)
1137 {
1138         static struct mlxsw_sp_fib_node *fib_node;
1139         const struct mlxsw_sp_ipip_ops *ipip_ops;
1140         struct mlxsw_sp_fib_entry *fib_entry;
1141         unsigned char saddr_prefix_len;
1142         union mlxsw_sp_l3addr saddr;
1143         struct mlxsw_sp_fib *ul_fib;
1144         struct mlxsw_sp_vr *ul_vr;
1145         const void *saddrp;
1146         size_t saddr_len;
1147         u32 ul_tb_id;
1148         u32 saddr4;
1149
1150         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1151
1152         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1153         ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1154         if (!ul_vr)
1155                 return NULL;
1156
1157         ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1158         saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1159                                            ipip_entry->ol_dev);
1160
1161         switch (ipip_ops->ul_proto) {
1162         case MLXSW_SP_L3_PROTO_IPV4:
1163                 saddr4 = be32_to_cpu(saddr.addr4);
1164                 saddrp = &saddr4;
1165                 saddr_len = 4;
1166                 saddr_prefix_len = 32;
1167                 break;
1168         case MLXSW_SP_L3_PROTO_IPV6:
1169                 WARN_ON(1);
1170                 return NULL;
1171         }
1172
1173         fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1174                                             saddr_prefix_len);
1175         if (!fib_node || list_empty(&fib_node->entry_list))
1176                 return NULL;
1177
1178         fib_entry = list_first_entry(&fib_node->entry_list,
1179                                      struct mlxsw_sp_fib_entry, list);
1180         if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1181                 return NULL;
1182
1183         return fib_entry;
1184 }
1185
1186 static struct mlxsw_sp_ipip_entry *
1187 mlxsw_sp_ipip_entry_get(struct mlxsw_sp *mlxsw_sp,
1188                         enum mlxsw_sp_ipip_type ipipt,
1189                         struct net_device *ol_dev)
1190 {
1191         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1192         struct mlxsw_sp_router *router = mlxsw_sp->router;
1193         struct mlxsw_sp_fib_entry *decap_fib_entry;
1194         struct mlxsw_sp_ipip_entry *ipip_entry;
1195         enum mlxsw_sp_l3proto ul_proto;
1196         union mlxsw_sp_l3addr saddr;
1197
1198         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1199                             ipip_list_node) {
1200                 if (ipip_entry->ol_dev == ol_dev)
1201                         goto inc_ref_count;
1202
1203                 /* The configuration where several tunnels have the same local
1204                  * address in the same underlay table needs special treatment in
1205                  * the HW. That is currently not implemented in the driver.
1206                  */
1207                 ul_proto = router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1208                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1209                 if (mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1210                                                       ul_tb_id, ipip_entry))
1211                         return ERR_PTR(-EEXIST);
1212         }
1213
1214         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1215         if (IS_ERR(ipip_entry))
1216                 return ipip_entry;
1217
1218         decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1219         if (decap_fib_entry)
1220                 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1221                                                   decap_fib_entry);
1222
1223         list_add_tail(&ipip_entry->ipip_list_node,
1224                       &mlxsw_sp->router->ipip_list);
1225
1226 inc_ref_count:
1227         ++ipip_entry->ref_count;
1228         return ipip_entry;
1229 }
1230
1231 static void
1232 mlxsw_sp_ipip_entry_put(struct mlxsw_sp *mlxsw_sp,
1233                         struct mlxsw_sp_ipip_entry *ipip_entry)
1234 {
1235         if (--ipip_entry->ref_count == 0) {
1236                 list_del(&ipip_entry->ipip_list_node);
1237                 if (ipip_entry->decap_fib_entry)
1238                         mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1239                 mlxsw_sp_ipip_entry_destroy(ipip_entry);
1240         }
1241 }
1242
1243 static bool
1244 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1245                                   const struct net_device *ul_dev,
1246                                   enum mlxsw_sp_l3proto ul_proto,
1247                                   union mlxsw_sp_l3addr ul_dip,
1248                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1249 {
1250         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1251         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1252         struct net_device *ipip_ul_dev;
1253
1254         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1255                 return false;
1256
1257         ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev);
1258         return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1259                                                  ul_tb_id, ipip_entry) &&
1260                (!ipip_ul_dev || ipip_ul_dev == ul_dev);
1261 }
1262
1263 /* Given decap parameters, find the corresponding IPIP entry. */
1264 static struct mlxsw_sp_ipip_entry *
1265 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp,
1266                                   const struct net_device *ul_dev,
1267                                   enum mlxsw_sp_l3proto ul_proto,
1268                                   union mlxsw_sp_l3addr ul_dip)
1269 {
1270         struct mlxsw_sp_ipip_entry *ipip_entry;
1271
1272         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1273                             ipip_list_node)
1274                 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1275                                                       ul_proto, ul_dip,
1276                                                       ipip_entry))
1277                         return ipip_entry;
1278
1279         return NULL;
1280 }
1281
1282 struct mlxsw_sp_neigh_key {
1283         struct neighbour *n;
1284 };
1285
1286 struct mlxsw_sp_neigh_entry {
1287         struct list_head rif_list_node;
1288         struct rhash_head ht_node;
1289         struct mlxsw_sp_neigh_key key;
1290         u16 rif;
1291         bool connected;
1292         unsigned char ha[ETH_ALEN];
1293         struct list_head nexthop_list; /* list of nexthops using
1294                                         * this neigh entry
1295                                         */
1296         struct list_head nexthop_neighs_list_node;
1297         unsigned int counter_index;
1298         bool counter_valid;
1299 };
1300
1301 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
1302         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
1303         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
1304         .key_len = sizeof(struct mlxsw_sp_neigh_key),
1305 };
1306
1307 struct mlxsw_sp_neigh_entry *
1308 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
1309                         struct mlxsw_sp_neigh_entry *neigh_entry)
1310 {
1311         if (!neigh_entry) {
1312                 if (list_empty(&rif->neigh_list))
1313                         return NULL;
1314                 else
1315                         return list_first_entry(&rif->neigh_list,
1316                                                 typeof(*neigh_entry),
1317                                                 rif_list_node);
1318         }
1319         if (neigh_entry->rif_list_node.next == &rif->neigh_list)
1320                 return NULL;
1321         return list_next_entry(neigh_entry, rif_list_node);
1322 }
1323
1324 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
1325 {
1326         return neigh_entry->key.n->tbl->family;
1327 }
1328
1329 unsigned char *
1330 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
1331 {
1332         return neigh_entry->ha;
1333 }
1334
1335 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1336 {
1337         struct neighbour *n;
1338
1339         n = neigh_entry->key.n;
1340         return ntohl(*((__be32 *) n->primary_key));
1341 }
1342
1343 struct in6_addr *
1344 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1345 {
1346         struct neighbour *n;
1347
1348         n = neigh_entry->key.n;
1349         return (struct in6_addr *) &n->primary_key;
1350 }
1351
1352 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
1353                                struct mlxsw_sp_neigh_entry *neigh_entry,
1354                                u64 *p_counter)
1355 {
1356         if (!neigh_entry->counter_valid)
1357                 return -EINVAL;
1358
1359         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
1360                                          p_counter, NULL);
1361 }
1362
1363 static struct mlxsw_sp_neigh_entry *
1364 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
1365                            u16 rif)
1366 {
1367         struct mlxsw_sp_neigh_entry *neigh_entry;
1368
1369         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
1370         if (!neigh_entry)
1371                 return NULL;
1372
1373         neigh_entry->key.n = n;
1374         neigh_entry->rif = rif;
1375         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
1376
1377         return neigh_entry;
1378 }
1379
1380 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
1381 {
1382         kfree(neigh_entry);
1383 }
1384
1385 static int
1386 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
1387                             struct mlxsw_sp_neigh_entry *neigh_entry)
1388 {
1389         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
1390                                       &neigh_entry->ht_node,
1391                                       mlxsw_sp_neigh_ht_params);
1392 }
1393
1394 static void
1395 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
1396                             struct mlxsw_sp_neigh_entry *neigh_entry)
1397 {
1398         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
1399                                &neigh_entry->ht_node,
1400                                mlxsw_sp_neigh_ht_params);
1401 }
1402
1403 static bool
1404 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
1405                                     struct mlxsw_sp_neigh_entry *neigh_entry)
1406 {
1407         struct devlink *devlink;
1408         const char *table_name;
1409
1410         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
1411         case AF_INET:
1412                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
1413                 break;
1414         case AF_INET6:
1415                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
1416                 break;
1417         default:
1418                 WARN_ON(1);
1419                 return false;
1420         }
1421
1422         devlink = priv_to_devlink(mlxsw_sp->core);
1423         return devlink_dpipe_table_counter_enabled(devlink, table_name);
1424 }
1425
1426 static void
1427 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
1428                              struct mlxsw_sp_neigh_entry *neigh_entry)
1429 {
1430         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
1431                 return;
1432
1433         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
1434                 return;
1435
1436         neigh_entry->counter_valid = true;
1437 }
1438
1439 static void
1440 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
1441                             struct mlxsw_sp_neigh_entry *neigh_entry)
1442 {
1443         if (!neigh_entry->counter_valid)
1444                 return;
1445         mlxsw_sp_flow_counter_free(mlxsw_sp,
1446                                    neigh_entry->counter_index);
1447         neigh_entry->counter_valid = false;
1448 }
1449
1450 static struct mlxsw_sp_neigh_entry *
1451 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1452 {
1453         struct mlxsw_sp_neigh_entry *neigh_entry;
1454         struct mlxsw_sp_rif *rif;
1455         int err;
1456
1457         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
1458         if (!rif)
1459                 return ERR_PTR(-EINVAL);
1460
1461         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
1462         if (!neigh_entry)
1463                 return ERR_PTR(-ENOMEM);
1464
1465         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
1466         if (err)
1467                 goto err_neigh_entry_insert;
1468
1469         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1470         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
1471
1472         return neigh_entry;
1473
1474 err_neigh_entry_insert:
1475         mlxsw_sp_neigh_entry_free(neigh_entry);
1476         return ERR_PTR(err);
1477 }
1478
1479 static void
1480 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1481                              struct mlxsw_sp_neigh_entry *neigh_entry)
1482 {
1483         list_del(&neigh_entry->rif_list_node);
1484         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1485         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
1486         mlxsw_sp_neigh_entry_free(neigh_entry);
1487 }
1488
1489 static struct mlxsw_sp_neigh_entry *
1490 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1491 {
1492         struct mlxsw_sp_neigh_key key;
1493
1494         key.n = n;
1495         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
1496                                       &key, mlxsw_sp_neigh_ht_params);
1497 }
1498
1499 static void
1500 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
1501 {
1502         unsigned long interval;
1503
1504 #if IS_ENABLED(CONFIG_IPV6)
1505         interval = min_t(unsigned long,
1506                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
1507                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
1508 #else
1509         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
1510 #endif
1511         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
1512 }
1513
1514 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1515                                                    char *rauhtd_pl,
1516                                                    int ent_index)
1517 {
1518         struct net_device *dev;
1519         struct neighbour *n;
1520         __be32 dipn;
1521         u32 dip;
1522         u16 rif;
1523
1524         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
1525
1526         if (!mlxsw_sp->router->rifs[rif]) {
1527                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1528                 return;
1529         }
1530
1531         dipn = htonl(dip);
1532         dev = mlxsw_sp->router->rifs[rif]->dev;
1533         n = neigh_lookup(&arp_tbl, &dipn, dev);
1534         if (!n) {
1535                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
1536                            &dip);
1537                 return;
1538         }
1539
1540         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
1541         neigh_event_send(n, NULL);
1542         neigh_release(n);
1543 }
1544
1545 #if IS_ENABLED(CONFIG_IPV6)
1546 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1547                                                    char *rauhtd_pl,
1548                                                    int rec_index)
1549 {
1550         struct net_device *dev;
1551         struct neighbour *n;
1552         struct in6_addr dip;
1553         u16 rif;
1554
1555         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
1556                                          (char *) &dip);
1557
1558         if (!mlxsw_sp->router->rifs[rif]) {
1559                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1560                 return;
1561         }
1562
1563         dev = mlxsw_sp->router->rifs[rif]->dev;
1564         n = neigh_lookup(&nd_tbl, &dip, dev);
1565         if (!n) {
1566                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
1567                            &dip);
1568                 return;
1569         }
1570
1571         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1572         neigh_event_send(n, NULL);
1573         neigh_release(n);
1574 }
1575 #else
1576 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1577                                                    char *rauhtd_pl,
1578                                                    int rec_index)
1579 {
1580 }
1581 #endif
1582
1583 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1584                                                    char *rauhtd_pl,
1585                                                    int rec_index)
1586 {
1587         u8 num_entries;
1588         int i;
1589
1590         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1591                                                                 rec_index);
1592         /* Hardware starts counting at 0, so add 1. */
1593         num_entries++;
1594
1595         /* Each record consists of several neighbour entries. */
1596         for (i = 0; i < num_entries; i++) {
1597                 int ent_index;
1598
1599                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1600                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1601                                                        ent_index);
1602         }
1603
1604 }
1605
1606 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1607                                                    char *rauhtd_pl,
1608                                                    int rec_index)
1609 {
1610         /* One record contains one entry. */
1611         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1612                                                rec_index);
1613 }
1614
1615 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1616                                               char *rauhtd_pl, int rec_index)
1617 {
1618         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1619         case MLXSW_REG_RAUHTD_TYPE_IPV4:
1620                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1621                                                        rec_index);
1622                 break;
1623         case MLXSW_REG_RAUHTD_TYPE_IPV6:
1624                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1625                                                        rec_index);
1626                 break;
1627         }
1628 }
1629
1630 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1631 {
1632         u8 num_rec, last_rec_index, num_entries;
1633
1634         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1635         last_rec_index = num_rec - 1;
1636
1637         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1638                 return false;
1639         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1640             MLXSW_REG_RAUHTD_TYPE_IPV6)
1641                 return true;
1642
1643         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1644                                                                 last_rec_index);
1645         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1646                 return true;
1647         return false;
1648 }
1649
1650 static int
1651 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1652                                        char *rauhtd_pl,
1653                                        enum mlxsw_reg_rauhtd_type type)
1654 {
1655         int i, num_rec;
1656         int err;
1657
1658         /* Make sure the neighbour's netdev isn't removed in the
1659          * process.
1660          */
1661         rtnl_lock();
1662         do {
1663                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
1664                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1665                                       rauhtd_pl);
1666                 if (err) {
1667                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1668                         break;
1669                 }
1670                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1671                 for (i = 0; i < num_rec; i++)
1672                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1673                                                           i);
1674         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1675         rtnl_unlock();
1676
1677         return err;
1678 }
1679
1680 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1681 {
1682         enum mlxsw_reg_rauhtd_type type;
1683         char *rauhtd_pl;
1684         int err;
1685
1686         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1687         if (!rauhtd_pl)
1688                 return -ENOMEM;
1689
1690         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1691         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1692         if (err)
1693                 goto out;
1694
1695         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1696         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1697 out:
1698         kfree(rauhtd_pl);
1699         return err;
1700 }
1701
1702 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1703 {
1704         struct mlxsw_sp_neigh_entry *neigh_entry;
1705
1706         /* Take RTNL mutex here to prevent lists from changes */
1707         rtnl_lock();
1708         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1709                             nexthop_neighs_list_node)
1710                 /* If this neigh have nexthops, make the kernel think this neigh
1711                  * is active regardless of the traffic.
1712                  */
1713                 neigh_event_send(neigh_entry->key.n, NULL);
1714         rtnl_unlock();
1715 }
1716
1717 static void
1718 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1719 {
1720         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1721
1722         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1723                                msecs_to_jiffies(interval));
1724 }
1725
1726 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1727 {
1728         struct mlxsw_sp_router *router;
1729         int err;
1730
1731         router = container_of(work, struct mlxsw_sp_router,
1732                               neighs_update.dw.work);
1733         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1734         if (err)
1735                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1736
1737         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1738
1739         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1740 }
1741
1742 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1743 {
1744         struct mlxsw_sp_neigh_entry *neigh_entry;
1745         struct mlxsw_sp_router *router;
1746
1747         router = container_of(work, struct mlxsw_sp_router,
1748                               nexthop_probe_dw.work);
1749         /* Iterate over nexthop neighbours, find those who are unresolved and
1750          * send arp on them. This solves the chicken-egg problem when
1751          * the nexthop wouldn't get offloaded until the neighbor is resolved
1752          * but it wouldn't get resolved ever in case traffic is flowing in HW
1753          * using different nexthop.
1754          *
1755          * Take RTNL mutex here to prevent lists from changes.
1756          */
1757         rtnl_lock();
1758         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1759                             nexthop_neighs_list_node)
1760                 if (!neigh_entry->connected)
1761                         neigh_event_send(neigh_entry->key.n, NULL);
1762         rtnl_unlock();
1763
1764         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1765                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1766 }
1767
1768 static void
1769 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1770                               struct mlxsw_sp_neigh_entry *neigh_entry,
1771                               bool removing);
1772
1773 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1774 {
1775         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1776                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1777 }
1778
1779 static void
1780 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1781                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1782                                 enum mlxsw_reg_rauht_op op)
1783 {
1784         struct neighbour *n = neigh_entry->key.n;
1785         u32 dip = ntohl(*((__be32 *) n->primary_key));
1786         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1787
1788         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1789                               dip);
1790         if (neigh_entry->counter_valid)
1791                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1792                                              neigh_entry->counter_index);
1793         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1794 }
1795
1796 static void
1797 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1798                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1799                                 enum mlxsw_reg_rauht_op op)
1800 {
1801         struct neighbour *n = neigh_entry->key.n;
1802         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1803         const char *dip = n->primary_key;
1804
1805         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1806                               dip);
1807         if (neigh_entry->counter_valid)
1808                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1809                                              neigh_entry->counter_index);
1810         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1811 }
1812
1813 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
1814 {
1815         struct neighbour *n = neigh_entry->key.n;
1816
1817         /* Packets with a link-local destination address are trapped
1818          * after LPM lookup and never reach the neighbour table, so
1819          * there is no need to program such neighbours to the device.
1820          */
1821         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1822             IPV6_ADDR_LINKLOCAL)
1823                 return true;
1824         return false;
1825 }
1826
1827 static void
1828 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1829                             struct mlxsw_sp_neigh_entry *neigh_entry,
1830                             bool adding)
1831 {
1832         if (!adding && !neigh_entry->connected)
1833                 return;
1834         neigh_entry->connected = adding;
1835         if (neigh_entry->key.n->tbl->family == AF_INET) {
1836                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1837                                                 mlxsw_sp_rauht_op(adding));
1838         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
1839                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
1840                         return;
1841                 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1842                                                 mlxsw_sp_rauht_op(adding));
1843         } else {
1844                 WARN_ON_ONCE(1);
1845         }
1846 }
1847
1848 void
1849 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
1850                                     struct mlxsw_sp_neigh_entry *neigh_entry,
1851                                     bool adding)
1852 {
1853         if (adding)
1854                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1855         else
1856                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1857         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
1858 }
1859
1860 struct mlxsw_sp_neigh_event_work {
1861         struct work_struct work;
1862         struct mlxsw_sp *mlxsw_sp;
1863         struct neighbour *n;
1864 };
1865
1866 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1867 {
1868         struct mlxsw_sp_neigh_event_work *neigh_work =
1869                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1870         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1871         struct mlxsw_sp_neigh_entry *neigh_entry;
1872         struct neighbour *n = neigh_work->n;
1873         unsigned char ha[ETH_ALEN];
1874         bool entry_connected;
1875         u8 nud_state, dead;
1876
1877         /* If these parameters are changed after we release the lock,
1878          * then we are guaranteed to receive another event letting us
1879          * know about it.
1880          */
1881         read_lock_bh(&n->lock);
1882         memcpy(ha, n->ha, ETH_ALEN);
1883         nud_state = n->nud_state;
1884         dead = n->dead;
1885         read_unlock_bh(&n->lock);
1886
1887         rtnl_lock();
1888         entry_connected = nud_state & NUD_VALID && !dead;
1889         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1890         if (!entry_connected && !neigh_entry)
1891                 goto out;
1892         if (!neigh_entry) {
1893                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1894                 if (IS_ERR(neigh_entry))
1895                         goto out;
1896         }
1897
1898         memcpy(neigh_entry->ha, ha, ETH_ALEN);
1899         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1900         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1901
1902         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1903                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1904
1905 out:
1906         rtnl_unlock();
1907         neigh_release(n);
1908         kfree(neigh_work);
1909 }
1910
1911 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1912                                    unsigned long event, void *ptr)
1913 {
1914         struct mlxsw_sp_neigh_event_work *neigh_work;
1915         struct mlxsw_sp_port *mlxsw_sp_port;
1916         struct mlxsw_sp *mlxsw_sp;
1917         unsigned long interval;
1918         struct neigh_parms *p;
1919         struct neighbour *n;
1920
1921         switch (event) {
1922         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1923                 p = ptr;
1924
1925                 /* We don't care about changes in the default table. */
1926                 if (!p->dev || (p->tbl->family != AF_INET &&
1927                                 p->tbl->family != AF_INET6))
1928                         return NOTIFY_DONE;
1929
1930                 /* We are in atomic context and can't take RTNL mutex,
1931                  * so use RCU variant to walk the device chain.
1932                  */
1933                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1934                 if (!mlxsw_sp_port)
1935                         return NOTIFY_DONE;
1936
1937                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1938                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1939                 mlxsw_sp->router->neighs_update.interval = interval;
1940
1941                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1942                 break;
1943         case NETEVENT_NEIGH_UPDATE:
1944                 n = ptr;
1945
1946                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
1947                         return NOTIFY_DONE;
1948
1949                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1950                 if (!mlxsw_sp_port)
1951                         return NOTIFY_DONE;
1952
1953                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1954                 if (!neigh_work) {
1955                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1956                         return NOTIFY_BAD;
1957                 }
1958
1959                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1960                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1961                 neigh_work->n = n;
1962
1963                 /* Take a reference to ensure the neighbour won't be
1964                  * destructed until we drop the reference in delayed
1965                  * work.
1966                  */
1967                 neigh_clone(n);
1968                 mlxsw_core_schedule_work(&neigh_work->work);
1969                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1970                 break;
1971         }
1972
1973         return NOTIFY_DONE;
1974 }
1975
1976 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1977 {
1978         int err;
1979
1980         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1981                               &mlxsw_sp_neigh_ht_params);
1982         if (err)
1983                 return err;
1984
1985         /* Initialize the polling interval according to the default
1986          * table.
1987          */
1988         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1989
1990         /* Create the delayed works for the activity_update */
1991         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1992                           mlxsw_sp_router_neighs_update_work);
1993         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1994                           mlxsw_sp_router_probe_unresolved_nexthops);
1995         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1996         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1997         return 0;
1998 }
1999
2000 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2001 {
2002         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2003         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2004         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2005 }
2006
2007 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2008                                          struct mlxsw_sp_rif *rif)
2009 {
2010         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2011
2012         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2013                                  rif_list_node) {
2014                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2015                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2016         }
2017 }
2018
2019 enum mlxsw_sp_nexthop_type {
2020         MLXSW_SP_NEXTHOP_TYPE_ETH,
2021         MLXSW_SP_NEXTHOP_TYPE_IPIP,
2022 };
2023
2024 struct mlxsw_sp_nexthop_key {
2025         struct fib_nh *fib_nh;
2026 };
2027
2028 struct mlxsw_sp_nexthop {
2029         struct list_head neigh_list_node; /* member of neigh entry list */
2030         struct list_head rif_list_node;
2031         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
2032                                                 * this belongs to
2033                                                 */
2034         struct rhash_head ht_node;
2035         struct mlxsw_sp_nexthop_key key;
2036         unsigned char gw_addr[sizeof(struct in6_addr)];
2037         int ifindex;
2038         struct mlxsw_sp_rif *rif;
2039         u8 should_offload:1, /* set indicates this neigh is connected and
2040                               * should be put to KVD linear area of this group.
2041                               */
2042            offloaded:1, /* set in case the neigh is actually put into
2043                          * KVD linear area of this group.
2044                          */
2045            update:1; /* set indicates that MAC of this neigh should be
2046                       * updated in HW
2047                       */
2048         enum mlxsw_sp_nexthop_type type;
2049         union {
2050                 struct mlxsw_sp_neigh_entry *neigh_entry;
2051                 struct mlxsw_sp_ipip_entry *ipip_entry;
2052         };
2053 };
2054
2055 struct mlxsw_sp_nexthop_group {
2056         void *priv;
2057         struct rhash_head ht_node;
2058         struct list_head fib_list; /* list of fib entries that use this group */
2059         struct neigh_table *neigh_tbl;
2060         u8 adj_index_valid:1,
2061            gateway:1; /* routes using the group use a gateway */
2062         u32 adj_index;
2063         u16 ecmp_size;
2064         u16 count;
2065         struct mlxsw_sp_nexthop nexthops[0];
2066 #define nh_rif  nexthops[0].rif
2067 };
2068
2069 static struct fib_info *
2070 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
2071 {
2072         return nh_grp->priv;
2073 }
2074
2075 struct mlxsw_sp_nexthop_group_cmp_arg {
2076         enum mlxsw_sp_l3proto proto;
2077         union {
2078                 struct fib_info *fi;
2079                 struct mlxsw_sp_fib6_entry *fib6_entry;
2080         };
2081 };
2082
2083 static bool
2084 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
2085                                     const struct in6_addr *gw, int ifindex)
2086 {
2087         int i;
2088
2089         for (i = 0; i < nh_grp->count; i++) {
2090                 const struct mlxsw_sp_nexthop *nh;
2091
2092                 nh = &nh_grp->nexthops[i];
2093                 if (nh->ifindex == ifindex &&
2094                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
2095                         return true;
2096         }
2097
2098         return false;
2099 }
2100
2101 static bool
2102 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
2103                             const struct mlxsw_sp_fib6_entry *fib6_entry)
2104 {
2105         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2106
2107         if (nh_grp->count != fib6_entry->nrt6)
2108                 return false;
2109
2110         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2111                 struct in6_addr *gw;
2112                 int ifindex;
2113
2114                 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
2115                 gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
2116                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
2117                         return false;
2118         }
2119
2120         return true;
2121 }
2122
2123 static int
2124 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
2125 {
2126         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
2127         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
2128
2129         switch (cmp_arg->proto) {
2130         case MLXSW_SP_L3_PROTO_IPV4:
2131                 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
2132         case MLXSW_SP_L3_PROTO_IPV6:
2133                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
2134                                                     cmp_arg->fib6_entry);
2135         default:
2136                 WARN_ON(1);
2137                 return 1;
2138         }
2139 }
2140
2141 static int
2142 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
2143 {
2144         return nh_grp->neigh_tbl->family;
2145 }
2146
2147 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
2148 {
2149         const struct mlxsw_sp_nexthop_group *nh_grp = data;
2150         const struct mlxsw_sp_nexthop *nh;
2151         struct fib_info *fi;
2152         unsigned int val;
2153         int i;
2154
2155         switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
2156         case AF_INET:
2157                 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
2158                 return jhash(&fi, sizeof(fi), seed);
2159         case AF_INET6:
2160                 val = nh_grp->count;
2161                 for (i = 0; i < nh_grp->count; i++) {
2162                         nh = &nh_grp->nexthops[i];
2163                         val ^= nh->ifindex;
2164                 }
2165                 return jhash(&val, sizeof(val), seed);
2166         default:
2167                 WARN_ON(1);
2168                 return 0;
2169         }
2170 }
2171
2172 static u32
2173 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
2174 {
2175         unsigned int val = fib6_entry->nrt6;
2176         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2177         struct net_device *dev;
2178
2179         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2180                 dev = mlxsw_sp_rt6->rt->dst.dev;
2181                 val ^= dev->ifindex;
2182         }
2183
2184         return jhash(&val, sizeof(val), seed);
2185 }
2186
2187 static u32
2188 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
2189 {
2190         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
2191
2192         switch (cmp_arg->proto) {
2193         case MLXSW_SP_L3_PROTO_IPV4:
2194                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
2195         case MLXSW_SP_L3_PROTO_IPV6:
2196                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
2197         default:
2198                 WARN_ON(1);
2199                 return 0;
2200         }
2201 }
2202
2203 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
2204         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
2205         .hashfn      = mlxsw_sp_nexthop_group_hash,
2206         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
2207         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
2208 };
2209
2210 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
2211                                          struct mlxsw_sp_nexthop_group *nh_grp)
2212 {
2213         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2214             !nh_grp->gateway)
2215                 return 0;
2216
2217         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
2218                                       &nh_grp->ht_node,
2219                                       mlxsw_sp_nexthop_group_ht_params);
2220 }
2221
2222 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
2223                                           struct mlxsw_sp_nexthop_group *nh_grp)
2224 {
2225         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2226             !nh_grp->gateway)
2227                 return;
2228
2229         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
2230                                &nh_grp->ht_node,
2231                                mlxsw_sp_nexthop_group_ht_params);
2232 }
2233
2234 static struct mlxsw_sp_nexthop_group *
2235 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
2236                                struct fib_info *fi)
2237 {
2238         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2239
2240         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
2241         cmp_arg.fi = fi;
2242         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2243                                       &cmp_arg,
2244                                       mlxsw_sp_nexthop_group_ht_params);
2245 }
2246
2247 static struct mlxsw_sp_nexthop_group *
2248 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
2249                                struct mlxsw_sp_fib6_entry *fib6_entry)
2250 {
2251         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2252
2253         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
2254         cmp_arg.fib6_entry = fib6_entry;
2255         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2256                                       &cmp_arg,
2257                                       mlxsw_sp_nexthop_group_ht_params);
2258 }
2259
2260 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
2261         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
2262         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
2263         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
2264 };
2265
2266 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
2267                                    struct mlxsw_sp_nexthop *nh)
2268 {
2269         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
2270                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
2271 }
2272
2273 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
2274                                     struct mlxsw_sp_nexthop *nh)
2275 {
2276         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
2277                                mlxsw_sp_nexthop_ht_params);
2278 }
2279
2280 static struct mlxsw_sp_nexthop *
2281 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
2282                         struct mlxsw_sp_nexthop_key key)
2283 {
2284         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
2285                                       mlxsw_sp_nexthop_ht_params);
2286 }
2287
2288 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
2289                                              const struct mlxsw_sp_fib *fib,
2290                                              u32 adj_index, u16 ecmp_size,
2291                                              u32 new_adj_index,
2292                                              u16 new_ecmp_size)
2293 {
2294         char raleu_pl[MLXSW_REG_RALEU_LEN];
2295
2296         mlxsw_reg_raleu_pack(raleu_pl,
2297                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
2298                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
2299                              new_ecmp_size);
2300         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
2301 }
2302
2303 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
2304                                           struct mlxsw_sp_nexthop_group *nh_grp,
2305                                           u32 old_adj_index, u16 old_ecmp_size)
2306 {
2307         struct mlxsw_sp_fib_entry *fib_entry;
2308         struct mlxsw_sp_fib *fib = NULL;
2309         int err;
2310
2311         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2312                 if (fib == fib_entry->fib_node->fib)
2313                         continue;
2314                 fib = fib_entry->fib_node->fib;
2315                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
2316                                                         old_adj_index,
2317                                                         old_ecmp_size,
2318                                                         nh_grp->adj_index,
2319                                                         nh_grp->ecmp_size);
2320                 if (err)
2321                         return err;
2322         }
2323         return 0;
2324 }
2325
2326 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
2327                                        struct mlxsw_sp_nexthop *nh)
2328 {
2329         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2330         char ratr_pl[MLXSW_REG_RATR_LEN];
2331
2332         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
2333                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
2334                             adj_index, neigh_entry->rif);
2335         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
2336         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
2337 }
2338
2339 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
2340                                         u32 adj_index,
2341                                         struct mlxsw_sp_nexthop *nh)
2342 {
2343         const struct mlxsw_sp_ipip_ops *ipip_ops;
2344
2345         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
2346         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
2347 }
2348
2349 static int
2350 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
2351                               struct mlxsw_sp_nexthop_group *nh_grp,
2352                               bool reallocate)
2353 {
2354         u32 adj_index = nh_grp->adj_index; /* base */
2355         struct mlxsw_sp_nexthop *nh;
2356         int i;
2357         int err;
2358
2359         for (i = 0; i < nh_grp->count; i++) {
2360                 nh = &nh_grp->nexthops[i];
2361
2362                 if (!nh->should_offload) {
2363                         nh->offloaded = 0;
2364                         continue;
2365                 }
2366
2367                 if (nh->update || reallocate) {
2368                         switch (nh->type) {
2369                         case MLXSW_SP_NEXTHOP_TYPE_ETH:
2370                                 err = mlxsw_sp_nexthop_mac_update
2371                                             (mlxsw_sp, adj_index, nh);
2372                                 break;
2373                         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2374                                 err = mlxsw_sp_nexthop_ipip_update
2375                                             (mlxsw_sp, adj_index, nh);
2376                                 break;
2377                         }
2378                         if (err)
2379                                 return err;
2380                         nh->update = 0;
2381                         nh->offloaded = 1;
2382                 }
2383                 adj_index++;
2384         }
2385         return 0;
2386 }
2387
2388 static bool
2389 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2390                                  const struct mlxsw_sp_fib_entry *fib_entry);
2391
2392 static int
2393 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
2394                                     struct mlxsw_sp_nexthop_group *nh_grp)
2395 {
2396         struct mlxsw_sp_fib_entry *fib_entry;
2397         int err;
2398
2399         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2400                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2401                                                       fib_entry))
2402                         continue;
2403                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2404                 if (err)
2405                         return err;
2406         }
2407         return 0;
2408 }
2409
2410 static void
2411 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2412                                    enum mlxsw_reg_ralue_op op, int err);
2413
2414 static void
2415 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
2416 {
2417         enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
2418         struct mlxsw_sp_fib_entry *fib_entry;
2419
2420         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2421                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2422                                                       fib_entry))
2423                         continue;
2424                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2425         }
2426 }
2427
2428 static void
2429 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
2430                                struct mlxsw_sp_nexthop_group *nh_grp)
2431 {
2432         struct mlxsw_sp_nexthop *nh;
2433         bool offload_change = false;
2434         u32 adj_index;
2435         u16 ecmp_size = 0;
2436         bool old_adj_index_valid;
2437         u32 old_adj_index;
2438         u16 old_ecmp_size;
2439         int i;
2440         int err;
2441
2442         if (!nh_grp->gateway) {
2443                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2444                 return;
2445         }
2446
2447         for (i = 0; i < nh_grp->count; i++) {
2448                 nh = &nh_grp->nexthops[i];
2449
2450                 if (nh->should_offload != nh->offloaded) {
2451                         offload_change = true;
2452                         if (nh->should_offload)
2453                                 nh->update = 1;
2454                 }
2455                 if (nh->should_offload)
2456                         ecmp_size++;
2457         }
2458         if (!offload_change) {
2459                 /* Nothing was added or removed, so no need to reallocate. Just
2460                  * update MAC on existing adjacency indexes.
2461                  */
2462                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false);
2463                 if (err) {
2464                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2465                         goto set_trap;
2466                 }
2467                 return;
2468         }
2469         if (!ecmp_size)
2470                 /* No neigh of this group is connected so we just set
2471                  * the trap and let everthing flow through kernel.
2472                  */
2473                 goto set_trap;
2474
2475         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
2476         if (err) {
2477                 /* We ran out of KVD linear space, just set the
2478                  * trap and let everything flow through kernel.
2479                  */
2480                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
2481                 goto set_trap;
2482         }
2483         old_adj_index_valid = nh_grp->adj_index_valid;
2484         old_adj_index = nh_grp->adj_index;
2485         old_ecmp_size = nh_grp->ecmp_size;
2486         nh_grp->adj_index_valid = 1;
2487         nh_grp->adj_index = adj_index;
2488         nh_grp->ecmp_size = ecmp_size;
2489         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true);
2490         if (err) {
2491                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2492                 goto set_trap;
2493         }
2494
2495         if (!old_adj_index_valid) {
2496                 /* The trap was set for fib entries, so we have to call
2497                  * fib entry update to unset it and use adjacency index.
2498                  */
2499                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2500                 if (err) {
2501                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
2502                         goto set_trap;
2503                 }
2504                 return;
2505         }
2506
2507         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
2508                                              old_adj_index, old_ecmp_size);
2509         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
2510         if (err) {
2511                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
2512                 goto set_trap;
2513         }
2514
2515         /* Offload state within the group changed, so update the flags. */
2516         mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
2517
2518         return;
2519
2520 set_trap:
2521         old_adj_index_valid = nh_grp->adj_index_valid;
2522         nh_grp->adj_index_valid = 0;
2523         for (i = 0; i < nh_grp->count; i++) {
2524                 nh = &nh_grp->nexthops[i];
2525                 nh->offloaded = 0;
2526         }
2527         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2528         if (err)
2529                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
2530         if (old_adj_index_valid)
2531                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
2532 }
2533
2534 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
2535                                             bool removing)
2536 {
2537         if (!removing)
2538                 nh->should_offload = 1;
2539         else if (nh->offloaded)
2540                 nh->should_offload = 0;
2541         nh->update = 1;
2542 }
2543
2544 static void
2545 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2546                               struct mlxsw_sp_neigh_entry *neigh_entry,
2547                               bool removing)
2548 {
2549         struct mlxsw_sp_nexthop *nh;
2550
2551         list_for_each_entry(nh, &neigh_entry->nexthop_list,
2552                             neigh_list_node) {
2553                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
2554                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2555         }
2556 }
2557
2558 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
2559                                       struct mlxsw_sp_rif *rif)
2560 {
2561         if (nh->rif)
2562                 return;
2563
2564         nh->rif = rif;
2565         list_add(&nh->rif_list_node, &rif->nexthop_list);
2566 }
2567
2568 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
2569 {
2570         if (!nh->rif)
2571                 return;
2572
2573         list_del(&nh->rif_list_node);
2574         nh->rif = NULL;
2575 }
2576
2577 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
2578                                        struct mlxsw_sp_nexthop *nh)
2579 {
2580         struct mlxsw_sp_neigh_entry *neigh_entry;
2581         struct neighbour *n;
2582         u8 nud_state, dead;
2583         int err;
2584
2585         if (!nh->nh_grp->gateway || nh->neigh_entry)
2586                 return 0;
2587
2588         /* Take a reference of neigh here ensuring that neigh would
2589          * not be destructed before the nexthop entry is finished.
2590          * The reference is taken either in neigh_lookup() or
2591          * in neigh_create() in case n is not found.
2592          */
2593         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
2594         if (!n) {
2595                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
2596                                  nh->rif->dev);
2597                 if (IS_ERR(n))
2598                         return PTR_ERR(n);
2599                 neigh_event_send(n, NULL);
2600         }
2601         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2602         if (!neigh_entry) {
2603                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2604                 if (IS_ERR(neigh_entry)) {
2605                         err = -EINVAL;
2606                         goto err_neigh_entry_create;
2607                 }
2608         }
2609
2610         /* If that is the first nexthop connected to that neigh, add to
2611          * nexthop_neighs_list
2612          */
2613         if (list_empty(&neigh_entry->nexthop_list))
2614                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
2615                               &mlxsw_sp->router->nexthop_neighs_list);
2616
2617         nh->neigh_entry = neigh_entry;
2618         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
2619         read_lock_bh(&n->lock);
2620         nud_state = n->nud_state;
2621         dead = n->dead;
2622         read_unlock_bh(&n->lock);
2623         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
2624
2625         return 0;
2626
2627 err_neigh_entry_create:
2628         neigh_release(n);
2629         return err;
2630 }
2631
2632 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
2633                                         struct mlxsw_sp_nexthop *nh)
2634 {
2635         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2636         struct neighbour *n;
2637
2638         if (!neigh_entry)
2639                 return;
2640         n = neigh_entry->key.n;
2641
2642         __mlxsw_sp_nexthop_neigh_update(nh, true);
2643         list_del(&nh->neigh_list_node);
2644         nh->neigh_entry = NULL;
2645
2646         /* If that is the last nexthop connected to that neigh, remove from
2647          * nexthop_neighs_list
2648          */
2649         if (list_empty(&neigh_entry->nexthop_list))
2650                 list_del(&neigh_entry->nexthop_neighs_list_node);
2651
2652         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2653                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2654
2655         neigh_release(n);
2656 }
2657
2658 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2659                                       const struct net_device *dev,
2660                                       enum mlxsw_sp_ipip_type *p_type)
2661 {
2662         struct mlxsw_sp_router *router = mlxsw_sp->router;
2663         const struct mlxsw_sp_ipip_ops *ipip_ops;
2664         enum mlxsw_sp_ipip_type ipipt;
2665
2666         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
2667                 ipip_ops = router->ipip_ops_arr[ipipt];
2668                 if (dev->type == ipip_ops->dev_type) {
2669                         if (p_type)
2670                                 *p_type = ipipt;
2671                         return true;
2672                 }
2673         }
2674         return false;
2675 }
2676
2677 static int mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
2678                                       enum mlxsw_sp_ipip_type ipipt,
2679                                       struct mlxsw_sp_nexthop *nh,
2680                                       struct net_device *ol_dev)
2681 {
2682         if (!nh->nh_grp->gateway || nh->ipip_entry)
2683                 return 0;
2684
2685         nh->ipip_entry = mlxsw_sp_ipip_entry_get(mlxsw_sp, ipipt, ol_dev);
2686         if (IS_ERR(nh->ipip_entry))
2687                 return PTR_ERR(nh->ipip_entry);
2688
2689         __mlxsw_sp_nexthop_neigh_update(nh, false);
2690         return 0;
2691 }
2692
2693 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
2694                                        struct mlxsw_sp_nexthop *nh)
2695 {
2696         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
2697
2698         if (!ipip_entry)
2699                 return;
2700
2701         __mlxsw_sp_nexthop_neigh_update(nh, true);
2702         mlxsw_sp_ipip_entry_put(mlxsw_sp, ipip_entry);
2703         nh->ipip_entry = NULL;
2704 }
2705
2706 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2707                                         const struct fib_nh *fib_nh,
2708                                         enum mlxsw_sp_ipip_type *p_ipipt)
2709 {
2710         struct net_device *dev = fib_nh->nh_dev;
2711
2712         return dev &&
2713                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
2714                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
2715 }
2716
2717 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
2718                                        struct mlxsw_sp_nexthop *nh)
2719 {
2720         switch (nh->type) {
2721         case MLXSW_SP_NEXTHOP_TYPE_ETH:
2722                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2723                 mlxsw_sp_nexthop_rif_fini(nh);
2724                 break;
2725         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2726                 mlxsw_sp_nexthop_rif_fini(nh);
2727                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
2728                 break;
2729         }
2730 }
2731
2732 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp,
2733                                        struct mlxsw_sp_nexthop *nh,
2734                                        struct fib_nh *fib_nh)
2735 {
2736         struct mlxsw_sp_router *router = mlxsw_sp->router;
2737         struct net_device *dev = fib_nh->nh_dev;
2738         enum mlxsw_sp_ipip_type ipipt;
2739         struct mlxsw_sp_rif *rif;
2740         int err;
2741
2742         if (mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fib_nh, &ipipt) &&
2743             router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
2744                                                      MLXSW_SP_L3_PROTO_IPV4)) {
2745                 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
2746                 err = mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
2747                 if (err)
2748                         return err;
2749                 mlxsw_sp_nexthop_rif_init(nh, &nh->ipip_entry->ol_lb->common);
2750                 return 0;
2751         }
2752
2753         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
2754         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
2755         if (!rif)
2756                 return 0;
2757
2758         mlxsw_sp_nexthop_rif_init(nh, rif);
2759         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2760         if (err)
2761                 goto err_neigh_init;
2762
2763         return 0;
2764
2765 err_neigh_init:
2766         mlxsw_sp_nexthop_rif_fini(nh);
2767         return err;
2768 }
2769
2770 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp,
2771                                         struct mlxsw_sp_nexthop *nh)
2772 {
2773         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2774 }
2775
2776 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
2777                                   struct mlxsw_sp_nexthop_group *nh_grp,
2778                                   struct mlxsw_sp_nexthop *nh,
2779                                   struct fib_nh *fib_nh)
2780 {
2781         struct net_device *dev = fib_nh->nh_dev;
2782         struct in_device *in_dev;
2783         int err;
2784
2785         nh->nh_grp = nh_grp;
2786         nh->key.fib_nh = fib_nh;
2787         memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
2788         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
2789         if (err)
2790                 return err;
2791
2792         if (!dev)
2793                 return 0;
2794
2795         in_dev = __in_dev_get_rtnl(dev);
2796         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
2797             fib_nh->nh_flags & RTNH_F_LINKDOWN)
2798                 return 0;
2799
2800         err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2801         if (err)
2802                 goto err_nexthop_neigh_init;
2803
2804         return 0;
2805
2806 err_nexthop_neigh_init:
2807         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2808         return err;
2809 }
2810
2811 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
2812                                    struct mlxsw_sp_nexthop *nh)
2813 {
2814         mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2815         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2816 }
2817
2818 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
2819                                     unsigned long event, struct fib_nh *fib_nh)
2820 {
2821         struct mlxsw_sp_nexthop_key key;
2822         struct mlxsw_sp_nexthop *nh;
2823
2824         if (mlxsw_sp->router->aborted)
2825                 return;
2826
2827         key.fib_nh = fib_nh;
2828         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
2829         if (WARN_ON_ONCE(!nh))
2830                 return;
2831
2832         switch (event) {
2833         case FIB_EVENT_NH_ADD:
2834                 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2835                 break;
2836         case FIB_EVENT_NH_DEL:
2837                 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2838                 break;
2839         }
2840
2841         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2842 }
2843
2844 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2845                                            struct mlxsw_sp_rif *rif)
2846 {
2847         struct mlxsw_sp_nexthop *nh, *tmp;
2848
2849         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
2850                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2851                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2852         }
2853 }
2854
2855 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
2856                                    const struct fib_info *fi)
2857 {
2858         return fi->fib_nh->nh_scope == RT_SCOPE_LINK ||
2859                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL);
2860 }
2861
2862 static struct mlxsw_sp_nexthop_group *
2863 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
2864 {
2865         struct mlxsw_sp_nexthop_group *nh_grp;
2866         struct mlxsw_sp_nexthop *nh;
2867         struct fib_nh *fib_nh;
2868         size_t alloc_size;
2869         int i;
2870         int err;
2871
2872         alloc_size = sizeof(*nh_grp) +
2873                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
2874         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
2875         if (!nh_grp)
2876                 return ERR_PTR(-ENOMEM);
2877         nh_grp->priv = fi;
2878         INIT_LIST_HEAD(&nh_grp->fib_list);
2879         nh_grp->neigh_tbl = &arp_tbl;
2880
2881         nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
2882         nh_grp->count = fi->fib_nhs;
2883         fib_info_hold(fi);
2884         for (i = 0; i < nh_grp->count; i++) {
2885                 nh = &nh_grp->nexthops[i];
2886                 fib_nh = &fi->fib_nh[i];
2887                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
2888                 if (err)
2889                         goto err_nexthop4_init;
2890         }
2891         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
2892         if (err)
2893                 goto err_nexthop_group_insert;
2894         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2895         return nh_grp;
2896
2897 err_nexthop_group_insert:
2898 err_nexthop4_init:
2899         for (i--; i >= 0; i--) {
2900                 nh = &nh_grp->nexthops[i];
2901                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2902         }
2903         fib_info_put(fi);
2904         kfree(nh_grp);
2905         return ERR_PTR(err);
2906 }
2907
2908 static void
2909 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
2910                                 struct mlxsw_sp_nexthop_group *nh_grp)
2911 {
2912         struct mlxsw_sp_nexthop *nh;
2913         int i;
2914
2915         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
2916         for (i = 0; i < nh_grp->count; i++) {
2917                 nh = &nh_grp->nexthops[i];
2918                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2919         }
2920         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2921         WARN_ON_ONCE(nh_grp->adj_index_valid);
2922         fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
2923         kfree(nh_grp);
2924 }
2925
2926 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
2927                                        struct mlxsw_sp_fib_entry *fib_entry,
2928                                        struct fib_info *fi)
2929 {
2930         struct mlxsw_sp_nexthop_group *nh_grp;
2931
2932         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
2933         if (!nh_grp) {
2934                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
2935                 if (IS_ERR(nh_grp))
2936                         return PTR_ERR(nh_grp);
2937         }
2938         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
2939         fib_entry->nh_group = nh_grp;
2940         return 0;
2941 }
2942
2943 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
2944                                         struct mlxsw_sp_fib_entry *fib_entry)
2945 {
2946         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2947
2948         list_del(&fib_entry->nexthop_group_node);
2949         if (!list_empty(&nh_grp->fib_list))
2950                 return;
2951         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
2952 }
2953
2954 static bool
2955 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2956 {
2957         struct mlxsw_sp_fib4_entry *fib4_entry;
2958
2959         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
2960                                   common);
2961         return !fib4_entry->tos;
2962 }
2963
2964 static bool
2965 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2966 {
2967         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
2968
2969         switch (fib_entry->fib_node->fib->proto) {
2970         case MLXSW_SP_L3_PROTO_IPV4:
2971                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
2972                         return false;
2973                 break;
2974         case MLXSW_SP_L3_PROTO_IPV6:
2975                 break;
2976         }
2977
2978         switch (fib_entry->type) {
2979         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2980                 return !!nh_group->adj_index_valid;
2981         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2982                 return !!nh_group->nh_rif;
2983         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
2984                 return true;
2985         default:
2986                 return false;
2987         }
2988 }
2989
2990 static struct mlxsw_sp_nexthop *
2991 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
2992                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
2993 {
2994         int i;
2995
2996         for (i = 0; i < nh_grp->count; i++) {
2997                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2998                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2999
3000                 if (nh->rif && nh->rif->dev == rt->dst.dev &&
3001                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
3002                                     &rt->rt6i_gateway))
3003                         return nh;
3004                 continue;
3005         }
3006
3007         return NULL;
3008 }
3009
3010 static void
3011 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3012 {
3013         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3014         int i;
3015
3016         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL ||
3017             fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) {
3018                 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3019                 return;
3020         }
3021
3022         for (i = 0; i < nh_grp->count; i++) {
3023                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3024
3025                 if (nh->offloaded)
3026                         nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3027                 else
3028                         nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3029         }
3030 }
3031
3032 static void
3033 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3034 {
3035         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3036         int i;
3037
3038         for (i = 0; i < nh_grp->count; i++) {
3039                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3040
3041                 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3042         }
3043 }
3044
3045 static void
3046 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3047 {
3048         struct mlxsw_sp_fib6_entry *fib6_entry;
3049         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3050
3051         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3052                                   common);
3053
3054         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
3055                 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3056                                  list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
3057                 return;
3058         }
3059
3060         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3061                 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3062                 struct mlxsw_sp_nexthop *nh;
3063
3064                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3065                 if (nh && nh->offloaded)
3066                         mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
3067                 else
3068                         mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
3069         }
3070 }
3071
3072 static void
3073 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3074 {
3075         struct mlxsw_sp_fib6_entry *fib6_entry;
3076         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3077
3078         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3079                                   common);
3080         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3081                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3082
3083                 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
3084         }
3085 }
3086
3087 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3088 {
3089         switch (fib_entry->fib_node->fib->proto) {
3090         case MLXSW_SP_L3_PROTO_IPV4:
3091                 mlxsw_sp_fib4_entry_offload_set(fib_entry);
3092                 break;
3093         case MLXSW_SP_L3_PROTO_IPV6:
3094                 mlxsw_sp_fib6_entry_offload_set(fib_entry);
3095                 break;
3096         }
3097 }
3098
3099 static void
3100 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3101 {
3102         switch (fib_entry->fib_node->fib->proto) {
3103         case MLXSW_SP_L3_PROTO_IPV4:
3104                 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
3105                 break;
3106         case MLXSW_SP_L3_PROTO_IPV6:
3107                 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
3108                 break;
3109         }
3110 }
3111
3112 static void
3113 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
3114                                    enum mlxsw_reg_ralue_op op, int err)
3115 {
3116         switch (op) {
3117         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
3118                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
3119         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
3120                 if (err)
3121                         return;
3122                 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
3123                         mlxsw_sp_fib_entry_offload_set(fib_entry);
3124                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
3125                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
3126                 return;
3127         default:
3128                 return;
3129         }
3130 }
3131
3132 static void
3133 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
3134                               const struct mlxsw_sp_fib_entry *fib_entry,
3135                               enum mlxsw_reg_ralue_op op)
3136 {
3137         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
3138         enum mlxsw_reg_ralxx_protocol proto;
3139         u32 *p_dip;
3140
3141         proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
3142
3143         switch (fib->proto) {
3144         case MLXSW_SP_L3_PROTO_IPV4:
3145                 p_dip = (u32 *) fib_entry->fib_node->key.addr;
3146                 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
3147                                       fib_entry->fib_node->key.prefix_len,
3148                                       *p_dip);
3149                 break;
3150         case MLXSW_SP_L3_PROTO_IPV6:
3151                 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
3152                                       fib_entry->fib_node->key.prefix_len,
3153                                       fib_entry->fib_node->key.addr);
3154                 break;
3155         }
3156 }
3157
3158 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
3159                                         struct mlxsw_sp_fib_entry *fib_entry,
3160                                         enum mlxsw_reg_ralue_op op)
3161 {
3162         char ralue_pl[MLXSW_REG_RALUE_LEN];
3163         enum mlxsw_reg_ralue_trap_action trap_action;
3164         u16 trap_id = 0;
3165         u32 adjacency_index = 0;
3166         u16 ecmp_size = 0;
3167
3168         /* In case the nexthop group adjacency index is valid, use it
3169          * with provided ECMP size. Otherwise, setup trap and pass
3170          * traffic to kernel.
3171          */
3172         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
3173                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
3174                 adjacency_index = fib_entry->nh_group->adj_index;
3175                 ecmp_size = fib_entry->nh_group->ecmp_size;
3176         } else {
3177                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
3178                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
3179         }
3180
3181         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3182         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
3183                                         adjacency_index, ecmp_size);
3184         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3185 }
3186
3187 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
3188                                        struct mlxsw_sp_fib_entry *fib_entry,
3189                                        enum mlxsw_reg_ralue_op op)
3190 {
3191         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
3192         enum mlxsw_reg_ralue_trap_action trap_action;
3193         char ralue_pl[MLXSW_REG_RALUE_LEN];
3194         u16 trap_id = 0;
3195         u16 rif_index = 0;
3196
3197         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
3198                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
3199                 rif_index = rif->rif_index;
3200         } else {
3201                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
3202                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
3203         }
3204
3205         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3206         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
3207                                        rif_index);
3208         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3209 }
3210
3211 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
3212                                       struct mlxsw_sp_fib_entry *fib_entry,
3213                                       enum mlxsw_reg_ralue_op op)
3214 {
3215         char ralue_pl[MLXSW_REG_RALUE_LEN];
3216
3217         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3218         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
3219         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3220 }
3221
3222 static int
3223 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
3224                                  struct mlxsw_sp_fib_entry *fib_entry,
3225                                  enum mlxsw_reg_ralue_op op)
3226 {
3227         struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
3228         const struct mlxsw_sp_ipip_ops *ipip_ops;
3229
3230         if (WARN_ON(!ipip_entry))
3231                 return -EINVAL;
3232
3233         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3234         return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op,
3235                                       fib_entry->decap.tunnel_index);
3236 }
3237
3238 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3239                                    struct mlxsw_sp_fib_entry *fib_entry,
3240                                    enum mlxsw_reg_ralue_op op)
3241 {
3242         switch (fib_entry->type) {
3243         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
3244                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
3245         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
3246                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
3247         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
3248                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
3249         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
3250                 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
3251                                                         fib_entry, op);
3252         }
3253         return -EINVAL;
3254 }
3255
3256 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3257                                  struct mlxsw_sp_fib_entry *fib_entry,
3258                                  enum mlxsw_reg_ralue_op op)
3259 {
3260         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
3261
3262         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
3263
3264         return err;
3265 }
3266
3267 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
3268                                      struct mlxsw_sp_fib_entry *fib_entry)
3269 {
3270         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3271                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
3272 }
3273
3274 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
3275                                   struct mlxsw_sp_fib_entry *fib_entry)
3276 {
3277         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3278                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
3279 }
3280
3281 static int
3282 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
3283                              const struct fib_entry_notifier_info *fen_info,
3284                              struct mlxsw_sp_fib_entry *fib_entry)
3285 {
3286         union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
3287         struct net_device *dev = fen_info->fi->fib_dev;
3288         struct mlxsw_sp_ipip_entry *ipip_entry;
3289         struct fib_info *fi = fen_info->fi;
3290
3291         switch (fen_info->type) {
3292         case RTN_LOCAL:
3293                 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev,
3294                                                  MLXSW_SP_L3_PROTO_IPV4, dip);
3295                 if (ipip_entry) {
3296                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
3297                         return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
3298                                                              fib_entry,
3299                                                              ipip_entry);
3300                 }
3301                 /* fall through */
3302         case RTN_BROADCAST:
3303                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
3304                 return 0;
3305         case RTN_UNREACHABLE: /* fall through */
3306         case RTN_BLACKHOLE: /* fall through */
3307         case RTN_PROHIBIT:
3308                 /* Packets hitting these routes need to be trapped, but
3309                  * can do so with a lower priority than packets directed
3310                  * at the host, so use action type local instead of trap.
3311                  */
3312                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3313                 return 0;
3314         case RTN_UNICAST:
3315                 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
3316                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
3317                 else
3318                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3319                 return 0;
3320         default:
3321                 return -EINVAL;
3322         }
3323 }
3324
3325 static struct mlxsw_sp_fib4_entry *
3326 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
3327                            struct mlxsw_sp_fib_node *fib_node,
3328                            const struct fib_entry_notifier_info *fen_info)
3329 {
3330         struct mlxsw_sp_fib4_entry *fib4_entry;
3331         struct mlxsw_sp_fib_entry *fib_entry;
3332         int err;
3333
3334         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
3335         if (!fib4_entry)
3336                 return ERR_PTR(-ENOMEM);
3337         fib_entry = &fib4_entry->common;
3338
3339         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
3340         if (err)
3341                 goto err_fib4_entry_type_set;
3342
3343         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
3344         if (err)
3345                 goto err_nexthop4_group_get;
3346
3347         fib4_entry->prio = fen_info->fi->fib_priority;
3348         fib4_entry->tb_id = fen_info->tb_id;
3349         fib4_entry->type = fen_info->type;
3350         fib4_entry->tos = fen_info->tos;
3351
3352         fib_entry->fib_node = fib_node;
3353
3354         return fib4_entry;
3355
3356 err_nexthop4_group_get:
3357 err_fib4_entry_type_set:
3358         kfree(fib4_entry);
3359         return ERR_PTR(err);
3360 }
3361
3362 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
3363                                         struct mlxsw_sp_fib4_entry *fib4_entry)
3364 {
3365         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
3366         kfree(fib4_entry);
3367 }
3368
3369 static struct mlxsw_sp_fib4_entry *
3370 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
3371                            const struct fib_entry_notifier_info *fen_info)
3372 {
3373         struct mlxsw_sp_fib4_entry *fib4_entry;
3374         struct mlxsw_sp_fib_node *fib_node;
3375         struct mlxsw_sp_fib *fib;
3376         struct mlxsw_sp_vr *vr;
3377
3378         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
3379         if (!vr)
3380                 return NULL;
3381         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
3382
3383         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
3384                                             sizeof(fen_info->dst),
3385                                             fen_info->dst_len);
3386         if (!fib_node)
3387                 return NULL;
3388
3389         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3390                 if (fib4_entry->tb_id == fen_info->tb_id &&
3391                     fib4_entry->tos == fen_info->tos &&
3392                     fib4_entry->type == fen_info->type &&
3393                     mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
3394                     fen_info->fi) {
3395                         return fib4_entry;
3396                 }
3397         }
3398
3399         return NULL;
3400 }
3401
3402 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
3403         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
3404         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
3405         .key_len = sizeof(struct mlxsw_sp_fib_key),
3406         .automatic_shrinking = true,
3407 };
3408
3409 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
3410                                     struct mlxsw_sp_fib_node *fib_node)
3411 {
3412         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
3413                                       mlxsw_sp_fib_ht_params);
3414 }
3415
3416 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
3417                                      struct mlxsw_sp_fib_node *fib_node)
3418 {
3419         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
3420                                mlxsw_sp_fib_ht_params);
3421 }
3422
3423 static struct mlxsw_sp_fib_node *
3424 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
3425                          size_t addr_len, unsigned char prefix_len)
3426 {
3427         struct mlxsw_sp_fib_key key;
3428
3429         memset(&key, 0, sizeof(key));
3430         memcpy(key.addr, addr, addr_len);
3431         key.prefix_len = prefix_len;
3432         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
3433 }
3434
3435 static struct mlxsw_sp_fib_node *
3436 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
3437                          size_t addr_len, unsigned char prefix_len)
3438 {
3439         struct mlxsw_sp_fib_node *fib_node;
3440
3441         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
3442         if (!fib_node)
3443                 return NULL;
3444
3445         INIT_LIST_HEAD(&fib_node->entry_list);
3446         list_add(&fib_node->list, &fib->node_list);
3447         memcpy(fib_node->key.addr, addr, addr_len);
3448         fib_node->key.prefix_len = prefix_len;
3449
3450         return fib_node;
3451 }
3452
3453 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
3454 {
3455         list_del(&fib_node->list);
3456         WARN_ON(!list_empty(&fib_node->entry_list));
3457         kfree(fib_node);
3458 }
3459
3460 static bool
3461 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
3462                                  const struct mlxsw_sp_fib_entry *fib_entry)
3463 {
3464         return list_first_entry(&fib_node->entry_list,
3465                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
3466 }
3467
3468 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
3469                                       struct mlxsw_sp_fib *fib,
3470                                       struct mlxsw_sp_fib_node *fib_node)
3471 {
3472         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3473         struct mlxsw_sp_lpm_tree *lpm_tree;
3474         int err;
3475
3476         /* Since the tree is shared between all virtual routers we must
3477          * make sure it contains all the required prefix lengths. This
3478          * can be computed by either adding the new prefix length to the
3479          * existing prefix usage of a bound tree, or by aggregating the
3480          * prefix lengths across all virtual routers and adding the new
3481          * one as well.
3482          */
3483         if (fib->lpm_tree)
3484                 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
3485                                           &fib->lpm_tree->prefix_usage);
3486         else
3487                 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3488         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
3489
3490         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3491                                          fib->proto);
3492         if (IS_ERR(lpm_tree))
3493                 return PTR_ERR(lpm_tree);
3494
3495         if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
3496                 return 0;
3497
3498         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3499         if (err)
3500                 return err;
3501
3502         return 0;
3503 }
3504
3505 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
3506                                          struct mlxsw_sp_fib *fib)
3507 {
3508         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3509         struct mlxsw_sp_lpm_tree *lpm_tree;
3510
3511         /* Aggregate prefix lengths across all virtual routers to make
3512          * sure we only have used prefix lengths in the LPM tree.
3513          */
3514         mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3515         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3516                                          fib->proto);
3517         if (IS_ERR(lpm_tree))
3518                 goto err_tree_get;
3519         mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3520
3521 err_tree_get:
3522         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
3523                 return;
3524         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
3525         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
3526         fib->lpm_tree = NULL;
3527 }
3528
3529 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
3530 {
3531         unsigned char prefix_len = fib_node->key.prefix_len;
3532         struct mlxsw_sp_fib *fib = fib_node->fib;
3533
3534         if (fib->prefix_ref_count[prefix_len]++ == 0)
3535                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
3536 }
3537
3538 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
3539 {
3540         unsigned char prefix_len = fib_node->key.prefix_len;
3541         struct mlxsw_sp_fib *fib = fib_node->fib;
3542
3543         if (--fib->prefix_ref_count[prefix_len] == 0)
3544                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
3545 }
3546
3547 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
3548                                   struct mlxsw_sp_fib_node *fib_node,
3549                                   struct mlxsw_sp_fib *fib)
3550 {
3551         int err;
3552
3553         err = mlxsw_sp_fib_node_insert(fib, fib_node);
3554         if (err)
3555                 return err;
3556         fib_node->fib = fib;
3557
3558         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
3559         if (err)
3560                 goto err_fib_lpm_tree_link;
3561
3562         mlxsw_sp_fib_node_prefix_inc(fib_node);
3563
3564         return 0;
3565
3566 err_fib_lpm_tree_link:
3567         fib_node->fib = NULL;
3568         mlxsw_sp_fib_node_remove(fib, fib_node);
3569         return err;
3570 }
3571
3572 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
3573                                    struct mlxsw_sp_fib_node *fib_node)
3574 {
3575         struct mlxsw_sp_fib *fib = fib_node->fib;
3576
3577         mlxsw_sp_fib_node_prefix_dec(fib_node);
3578         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
3579         fib_node->fib = NULL;
3580         mlxsw_sp_fib_node_remove(fib, fib_node);
3581 }
3582
3583 static struct mlxsw_sp_fib_node *
3584 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
3585                       size_t addr_len, unsigned char prefix_len,
3586                       enum mlxsw_sp_l3proto proto)
3587 {
3588         struct mlxsw_sp_fib_node *fib_node;
3589         struct mlxsw_sp_fib *fib;
3590         struct mlxsw_sp_vr *vr;
3591         int err;
3592
3593         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
3594         if (IS_ERR(vr))
3595                 return ERR_CAST(vr);
3596         fib = mlxsw_sp_vr_fib(vr, proto);
3597
3598         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
3599         if (fib_node)
3600                 return fib_node;
3601
3602         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
3603         if (!fib_node) {
3604                 err = -ENOMEM;
3605                 goto err_fib_node_create;
3606         }
3607
3608         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
3609         if (err)
3610                 goto err_fib_node_init;
3611
3612         return fib_node;
3613
3614 err_fib_node_init:
3615         mlxsw_sp_fib_node_destroy(fib_node);
3616 err_fib_node_create:
3617         mlxsw_sp_vr_put(vr);
3618         return ERR_PTR(err);
3619 }
3620
3621 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
3622                                   struct mlxsw_sp_fib_node *fib_node)
3623 {
3624         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
3625
3626         if (!list_empty(&fib_node->entry_list))
3627                 return;
3628         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
3629         mlxsw_sp_fib_node_destroy(fib_node);
3630         mlxsw_sp_vr_put(vr);
3631 }
3632
3633 static struct mlxsw_sp_fib4_entry *
3634 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3635                               const struct mlxsw_sp_fib4_entry *new4_entry)
3636 {
3637         struct mlxsw_sp_fib4_entry *fib4_entry;
3638
3639         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3640                 if (fib4_entry->tb_id > new4_entry->tb_id)
3641                         continue;
3642                 if (fib4_entry->tb_id != new4_entry->tb_id)
3643                         break;
3644                 if (fib4_entry->tos > new4_entry->tos)
3645                         continue;
3646                 if (fib4_entry->prio >= new4_entry->prio ||
3647                     fib4_entry->tos < new4_entry->tos)
3648                         return fib4_entry;
3649         }
3650
3651         return NULL;
3652 }
3653
3654 static int
3655 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
3656                                struct mlxsw_sp_fib4_entry *new4_entry)
3657 {
3658         struct mlxsw_sp_fib_node *fib_node;
3659
3660         if (WARN_ON(!fib4_entry))
3661                 return -EINVAL;
3662
3663         fib_node = fib4_entry->common.fib_node;
3664         list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
3665                                  common.list) {
3666                 if (fib4_entry->tb_id != new4_entry->tb_id ||
3667                     fib4_entry->tos != new4_entry->tos ||
3668                     fib4_entry->prio != new4_entry->prio)
3669                         break;
3670         }
3671
3672         list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
3673         return 0;
3674 }
3675
3676 static int
3677 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
3678                                bool replace, bool append)
3679 {
3680         struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
3681         struct mlxsw_sp_fib4_entry *fib4_entry;
3682
3683         fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
3684
3685         if (append)
3686                 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
3687         if (replace && WARN_ON(!fib4_entry))
3688                 return -EINVAL;
3689
3690         /* Insert new entry before replaced one, so that we can later
3691          * remove the second.
3692          */
3693         if (fib4_entry) {
3694                 list_add_tail(&new4_entry->common.list,
3695                               &fib4_entry->common.list);
3696         } else {
3697                 struct mlxsw_sp_fib4_entry *last;
3698
3699                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3700                         if (new4_entry->tb_id > last->tb_id)
3701                                 break;
3702                         fib4_entry = last;
3703                 }
3704
3705                 if (fib4_entry)
3706                         list_add(&new4_entry->common.list,
3707                                  &fib4_entry->common.list);
3708                 else
3709                         list_add(&new4_entry->common.list,
3710                                  &fib_node->entry_list);
3711         }
3712
3713         return 0;
3714 }
3715
3716 static void
3717 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
3718 {
3719         list_del(&fib4_entry->common.list);
3720 }
3721
3722 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
3723                                        struct mlxsw_sp_fib_entry *fib_entry)
3724 {
3725         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3726
3727         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3728                 return 0;
3729
3730         /* To prevent packet loss, overwrite the previously offloaded
3731          * entry.
3732          */
3733         if (!list_is_singular(&fib_node->entry_list)) {
3734                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3735                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3736
3737                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
3738         }
3739
3740         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3741 }
3742
3743 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
3744                                         struct mlxsw_sp_fib_entry *fib_entry)
3745 {
3746         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3747
3748         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3749                 return;
3750
3751         /* Promote the next entry by overwriting the deleted entry */
3752         if (!list_is_singular(&fib_node->entry_list)) {
3753                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3754                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3755
3756                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
3757                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3758                 return;
3759         }
3760
3761         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
3762 }
3763
3764 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3765                                          struct mlxsw_sp_fib4_entry *fib4_entry,
3766                                          bool replace, bool append)
3767 {
3768         int err;
3769
3770         err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
3771         if (err)
3772                 return err;
3773
3774         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
3775         if (err)
3776                 goto err_fib_node_entry_add;
3777
3778         return 0;
3779
3780 err_fib_node_entry_add:
3781         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3782         return err;
3783 }
3784
3785 static void
3786 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3787                                 struct mlxsw_sp_fib4_entry *fib4_entry)
3788 {
3789         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
3790         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3791
3792         if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP)
3793                 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common);
3794 }
3795
3796 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
3797                                         struct mlxsw_sp_fib4_entry *fib4_entry,
3798                                         bool replace)
3799 {
3800         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
3801         struct mlxsw_sp_fib4_entry *replaced;
3802
3803         if (!replace)
3804                 return;
3805
3806         /* We inserted the new entry before replaced one */
3807         replaced = list_next_entry(fib4_entry, common.list);
3808
3809         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
3810         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
3811         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3812 }
3813
3814 static int
3815 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
3816                          const struct fib_entry_notifier_info *fen_info,
3817                          bool replace, bool append)
3818 {
3819         struct mlxsw_sp_fib4_entry *fib4_entry;
3820         struct mlxsw_sp_fib_node *fib_node;
3821         int err;
3822
3823         if (mlxsw_sp->router->aborted)
3824                 return 0;
3825
3826         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
3827                                          &fen_info->dst, sizeof(fen_info->dst),
3828                                          fen_info->dst_len,
3829                                          MLXSW_SP_L3_PROTO_IPV4);
3830         if (IS_ERR(fib_node)) {
3831                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
3832                 return PTR_ERR(fib_node);
3833         }
3834
3835         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
3836         if (IS_ERR(fib4_entry)) {
3837                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
3838                 err = PTR_ERR(fib4_entry);
3839                 goto err_fib4_entry_create;
3840         }
3841
3842         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
3843                                             append);
3844         if (err) {
3845                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
3846                 goto err_fib4_node_entry_link;
3847         }
3848
3849         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
3850
3851         return 0;
3852
3853 err_fib4_node_entry_link:
3854         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3855 err_fib4_entry_create:
3856         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3857         return err;
3858 }
3859
3860 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
3861                                      struct fib_entry_notifier_info *fen_info)
3862 {
3863         struct mlxsw_sp_fib4_entry *fib4_entry;
3864         struct mlxsw_sp_fib_node *fib_node;
3865
3866         if (mlxsw_sp->router->aborted)
3867                 return;
3868
3869         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
3870         if (WARN_ON(!fib4_entry))
3871                 return;
3872         fib_node = fib4_entry->common.fib_node;
3873
3874         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
3875         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3876         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3877 }
3878
3879 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
3880 {
3881         /* Packets with link-local destination IP arriving to the router
3882          * are trapped to the CPU, so no need to program specific routes
3883          * for them.
3884          */
3885         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
3886                 return true;
3887
3888         /* Multicast routes aren't supported, so ignore them. Neighbour
3889          * Discovery packets are specifically trapped.
3890          */
3891         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
3892                 return true;
3893
3894         /* Cloned routes are irrelevant in the forwarding path. */
3895         if (rt->rt6i_flags & RTF_CACHE)
3896                 return true;
3897
3898         return false;
3899 }
3900
3901 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
3902 {
3903         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3904
3905         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
3906         if (!mlxsw_sp_rt6)
3907                 return ERR_PTR(-ENOMEM);
3908
3909         /* In case of route replace, replaced route is deleted with
3910          * no notification. Take reference to prevent accessing freed
3911          * memory.
3912          */
3913         mlxsw_sp_rt6->rt = rt;
3914         rt6_hold(rt);
3915
3916         return mlxsw_sp_rt6;
3917 }
3918
3919 #if IS_ENABLED(CONFIG_IPV6)
3920 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3921 {
3922         rt6_release(rt);
3923 }
3924 #else
3925 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3926 {
3927 }
3928 #endif
3929
3930 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3931 {
3932         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
3933         kfree(mlxsw_sp_rt6);
3934 }
3935
3936 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
3937 {
3938         /* RTF_CACHE routes are ignored */
3939         return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
3940 }
3941
3942 static struct rt6_info *
3943 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
3944 {
3945         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3946                                 list)->rt;
3947 }
3948
3949 static struct mlxsw_sp_fib6_entry *
3950 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3951                                  const struct rt6_info *nrt, bool replace)
3952 {
3953         struct mlxsw_sp_fib6_entry *fib6_entry;
3954
3955         if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
3956                 return NULL;
3957
3958         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3959                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3960
3961                 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
3962                  * virtual router.
3963                  */
3964                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3965                         continue;
3966                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3967                         break;
3968                 if (rt->rt6i_metric < nrt->rt6i_metric)
3969                         continue;
3970                 if (rt->rt6i_metric == nrt->rt6i_metric &&
3971                     mlxsw_sp_fib6_rt_can_mp(rt))
3972                         return fib6_entry;
3973                 if (rt->rt6i_metric > nrt->rt6i_metric)
3974                         break;
3975         }
3976
3977         return NULL;
3978 }
3979
3980 static struct mlxsw_sp_rt6 *
3981 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
3982                             const struct rt6_info *rt)
3983 {
3984         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3985
3986         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3987                 if (mlxsw_sp_rt6->rt == rt)
3988                         return mlxsw_sp_rt6;
3989         }
3990
3991         return NULL;
3992 }
3993
3994 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3995                                         const struct rt6_info *rt,
3996                                         enum mlxsw_sp_ipip_type *ret)
3997 {
3998         return rt->dst.dev &&
3999                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->dst.dev, ret);
4000 }
4001
4002 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
4003                                        struct mlxsw_sp_nexthop_group *nh_grp,
4004                                        struct mlxsw_sp_nexthop *nh,
4005                                        const struct rt6_info *rt)
4006 {
4007         struct mlxsw_sp_router *router = mlxsw_sp->router;
4008         struct net_device *dev = rt->dst.dev;
4009         enum mlxsw_sp_ipip_type ipipt;
4010         struct mlxsw_sp_rif *rif;
4011         int err;
4012
4013         if (mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, &ipipt) &&
4014             router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
4015                                                      MLXSW_SP_L3_PROTO_IPV6)) {
4016                 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4017                 err = mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
4018                 if (err)
4019                         return err;
4020                 mlxsw_sp_nexthop_rif_init(nh, &nh->ipip_entry->ol_lb->common);
4021                 return 0;
4022         }
4023
4024         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4025         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4026         if (!rif)
4027                 return 0;
4028         mlxsw_sp_nexthop_rif_init(nh, rif);
4029
4030         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4031         if (err)
4032                 goto err_nexthop_neigh_init;
4033
4034         return 0;
4035
4036 err_nexthop_neigh_init:
4037         mlxsw_sp_nexthop_rif_fini(nh);
4038         return err;
4039 }
4040
4041 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
4042                                         struct mlxsw_sp_nexthop *nh)
4043 {
4044         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4045 }
4046
4047 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
4048                                   struct mlxsw_sp_nexthop_group *nh_grp,
4049                                   struct mlxsw_sp_nexthop *nh,
4050                                   const struct rt6_info *rt)
4051 {
4052         struct net_device *dev = rt->dst.dev;
4053
4054         nh->nh_grp = nh_grp;
4055         memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
4056
4057         if (!dev)
4058                 return 0;
4059         nh->ifindex = dev->ifindex;
4060
4061         return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt);
4062 }
4063
4064 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
4065                                    struct mlxsw_sp_nexthop *nh)
4066 {
4067         mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh);
4068 }
4069
4070 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4071                                     const struct rt6_info *rt)
4072 {
4073         return rt->rt6i_flags & RTF_GATEWAY ||
4074                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
4075 }
4076
4077 static struct mlxsw_sp_nexthop_group *
4078 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
4079                                struct mlxsw_sp_fib6_entry *fib6_entry)
4080 {
4081         struct mlxsw_sp_nexthop_group *nh_grp;
4082         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4083         struct mlxsw_sp_nexthop *nh;
4084         size_t alloc_size;
4085         int i = 0;
4086         int err;
4087
4088         alloc_size = sizeof(*nh_grp) +
4089                      fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
4090         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
4091         if (!nh_grp)
4092                 return ERR_PTR(-ENOMEM);
4093         INIT_LIST_HEAD(&nh_grp->fib_list);
4094 #if IS_ENABLED(CONFIG_IPV6)
4095         nh_grp->neigh_tbl = &nd_tbl;
4096 #endif
4097         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
4098                                         struct mlxsw_sp_rt6, list);
4099         nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
4100         nh_grp->count = fib6_entry->nrt6;
4101         for (i = 0; i < nh_grp->count; i++) {
4102                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
4103
4104                 nh = &nh_grp->nexthops[i];
4105                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
4106                 if (err)
4107                         goto err_nexthop6_init;
4108                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
4109         }
4110
4111         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4112         if (err)
4113                 goto err_nexthop_group_insert;
4114
4115         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4116         return nh_grp;
4117
4118 err_nexthop_group_insert:
4119 err_nexthop6_init:
4120         for (i--; i >= 0; i--) {
4121                 nh = &nh_grp->nexthops[i];
4122                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4123         }
4124         kfree(nh_grp);
4125         return ERR_PTR(err);
4126 }
4127
4128 static void
4129 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
4130                                 struct mlxsw_sp_nexthop_group *nh_grp)
4131 {
4132         struct mlxsw_sp_nexthop *nh;
4133         int i = nh_grp->count;
4134
4135         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4136         for (i--; i >= 0; i--) {
4137                 nh = &nh_grp->nexthops[i];
4138                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4139         }
4140         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4141         WARN_ON(nh_grp->adj_index_valid);
4142         kfree(nh_grp);
4143 }
4144
4145 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
4146                                        struct mlxsw_sp_fib6_entry *fib6_entry)
4147 {
4148         struct mlxsw_sp_nexthop_group *nh_grp;
4149
4150         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
4151         if (!nh_grp) {
4152                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
4153                 if (IS_ERR(nh_grp))
4154                         return PTR_ERR(nh_grp);
4155         }
4156
4157         list_add_tail(&fib6_entry->common.nexthop_group_node,
4158                       &nh_grp->fib_list);
4159         fib6_entry->common.nh_group = nh_grp;
4160
4161         return 0;
4162 }
4163
4164 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
4165                                         struct mlxsw_sp_fib_entry *fib_entry)
4166 {
4167         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4168
4169         list_del(&fib_entry->nexthop_group_node);
4170         if (!list_empty(&nh_grp->fib_list))
4171                 return;
4172         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
4173 }
4174
4175 static int
4176 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
4177                                struct mlxsw_sp_fib6_entry *fib6_entry)
4178 {
4179         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
4180         int err;
4181
4182         fib6_entry->common.nh_group = NULL;
4183         list_del(&fib6_entry->common.nexthop_group_node);
4184
4185         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4186         if (err)
4187                 goto err_nexthop6_group_get;
4188
4189         /* In case this entry is offloaded, then the adjacency index
4190          * currently associated with it in the device's table is that
4191          * of the old group. Start using the new one instead.
4192          */
4193         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4194         if (err)
4195                 goto err_fib_node_entry_add;
4196
4197         if (list_empty(&old_nh_grp->fib_list))
4198                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
4199
4200         return 0;
4201
4202 err_fib_node_entry_add:
4203         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4204 err_nexthop6_group_get:
4205         list_add_tail(&fib6_entry->common.nexthop_group_node,
4206                       &old_nh_grp->fib_list);
4207         fib6_entry->common.nh_group = old_nh_grp;
4208         return err;
4209 }
4210
4211 static int
4212 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
4213                                 struct mlxsw_sp_fib6_entry *fib6_entry,
4214                                 struct rt6_info *rt)
4215 {
4216         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4217         int err;
4218
4219         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4220         if (IS_ERR(mlxsw_sp_rt6))
4221                 return PTR_ERR(mlxsw_sp_rt6);
4222
4223         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4224         fib6_entry->nrt6++;
4225
4226         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4227         if (err)
4228                 goto err_nexthop6_group_update;
4229
4230         return 0;
4231
4232 err_nexthop6_group_update:
4233         fib6_entry->nrt6--;
4234         list_del(&mlxsw_sp_rt6->list);
4235         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4236         return err;
4237 }
4238
4239 static void
4240 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
4241                                 struct mlxsw_sp_fib6_entry *fib6_entry,
4242                                 struct rt6_info *rt)
4243 {
4244         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4245
4246         mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
4247         if (WARN_ON(!mlxsw_sp_rt6))
4248                 return;
4249
4250         fib6_entry->nrt6--;
4251         list_del(&mlxsw_sp_rt6->list);
4252         mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4253         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4254 }
4255
4256 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4257                                          struct mlxsw_sp_fib_entry *fib_entry,
4258                                          const struct rt6_info *rt)
4259 {
4260         /* Packets hitting RTF_REJECT routes need to be discarded by the
4261          * stack. We can rely on their destination device not having a
4262          * RIF (it's the loopback device) and can thus use action type
4263          * local, which will cause them to be trapped with a lower
4264          * priority than packets that need to be locally received.
4265          */
4266         if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
4267                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4268         else if (rt->rt6i_flags & RTF_REJECT)
4269                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4270         else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
4271                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4272         else
4273                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4274 }
4275
4276 static void
4277 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
4278 {
4279         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
4280
4281         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
4282                                  list) {
4283                 fib6_entry->nrt6--;
4284                 list_del(&mlxsw_sp_rt6->list);
4285                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4286         }
4287 }
4288
4289 static struct mlxsw_sp_fib6_entry *
4290 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
4291                            struct mlxsw_sp_fib_node *fib_node,
4292                            struct rt6_info *rt)
4293 {
4294         struct mlxsw_sp_fib6_entry *fib6_entry;
4295         struct mlxsw_sp_fib_entry *fib_entry;
4296         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4297         int err;
4298
4299         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
4300         if (!fib6_entry)
4301                 return ERR_PTR(-ENOMEM);
4302         fib_entry = &fib6_entry->common;
4303
4304         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4305         if (IS_ERR(mlxsw_sp_rt6)) {
4306                 err = PTR_ERR(mlxsw_sp_rt6);
4307                 goto err_rt6_create;
4308         }
4309
4310         mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt);
4311
4312         INIT_LIST_HEAD(&fib6_entry->rt6_list);
4313         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4314         fib6_entry->nrt6 = 1;
4315         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4316         if (err)
4317                 goto err_nexthop6_group_get;
4318
4319         fib_entry->fib_node = fib_node;
4320
4321         return fib6_entry;
4322
4323 err_nexthop6_group_get:
4324         list_del(&mlxsw_sp_rt6->list);
4325         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4326 err_rt6_create:
4327         kfree(fib6_entry);
4328         return ERR_PTR(err);
4329 }
4330
4331 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4332                                         struct mlxsw_sp_fib6_entry *fib6_entry)
4333 {
4334         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4335         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
4336         WARN_ON(fib6_entry->nrt6);
4337         kfree(fib6_entry);
4338 }
4339
4340 static struct mlxsw_sp_fib6_entry *
4341 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4342                               const struct rt6_info *nrt, bool replace)
4343 {
4344         struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
4345
4346         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4347                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4348
4349                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
4350                         continue;
4351                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
4352                         break;
4353                 if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
4354                         if (mlxsw_sp_fib6_rt_can_mp(rt) ==
4355                             mlxsw_sp_fib6_rt_can_mp(nrt))
4356                                 return fib6_entry;
4357                         if (mlxsw_sp_fib6_rt_can_mp(nrt))
4358                                 fallback = fallback ?: fib6_entry;
4359                 }
4360                 if (rt->rt6i_metric > nrt->rt6i_metric)
4361                         return fallback ?: fib6_entry;
4362         }
4363
4364         return fallback;
4365 }
4366
4367 static int
4368 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
4369                                bool replace)
4370 {
4371         struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
4372         struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
4373         struct mlxsw_sp_fib6_entry *fib6_entry;
4374
4375         fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
4376
4377         if (replace && WARN_ON(!fib6_entry))
4378                 return -EINVAL;
4379
4380         if (fib6_entry) {
4381                 list_add_tail(&new6_entry->common.list,
4382                               &fib6_entry->common.list);
4383         } else {
4384                 struct mlxsw_sp_fib6_entry *last;
4385
4386                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
4387                         struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
4388
4389                         if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
4390                                 break;
4391                         fib6_entry = last;
4392                 }
4393
4394                 if (fib6_entry)
4395                         list_add(&new6_entry->common.list,
4396                                  &fib6_entry->common.list);
4397                 else
4398                         list_add(&new6_entry->common.list,
4399                                  &fib_node->entry_list);
4400         }
4401
4402         return 0;
4403 }
4404
4405 static void
4406 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
4407 {
4408         list_del(&fib6_entry->common.list);
4409 }
4410
4411 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4412                                          struct mlxsw_sp_fib6_entry *fib6_entry,
4413                                          bool replace)
4414 {
4415         int err;
4416
4417         err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
4418         if (err)
4419                 return err;
4420
4421         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4422         if (err)
4423                 goto err_fib_node_entry_add;
4424
4425         return 0;
4426
4427 err_fib_node_entry_add:
4428         mlxsw_sp_fib6_node_list_remove(fib6_entry);
4429         return err;
4430 }
4431
4432 static void
4433 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4434                                 struct mlxsw_sp_fib6_entry *fib6_entry)
4435 {
4436         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
4437         mlxsw_sp_fib6_node_list_remove(fib6_entry);
4438 }
4439
4440 static struct mlxsw_sp_fib6_entry *
4441 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4442                            const struct rt6_info *rt)
4443 {
4444         struct mlxsw_sp_fib6_entry *fib6_entry;
4445         struct mlxsw_sp_fib_node *fib_node;
4446         struct mlxsw_sp_fib *fib;
4447         struct mlxsw_sp_vr *vr;
4448
4449         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
4450         if (!vr)
4451                 return NULL;
4452         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
4453
4454         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
4455                                             sizeof(rt->rt6i_dst.addr),
4456                                             rt->rt6i_dst.plen);
4457         if (!fib_node)
4458                 return NULL;
4459
4460         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4461                 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4462
4463                 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
4464                     rt->rt6i_metric == iter_rt->rt6i_metric &&
4465                     mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
4466                         return fib6_entry;
4467         }
4468
4469         return NULL;
4470 }
4471
4472 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
4473                                         struct mlxsw_sp_fib6_entry *fib6_entry,
4474                                         bool replace)
4475 {
4476         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
4477         struct mlxsw_sp_fib6_entry *replaced;
4478
4479         if (!replace)
4480                 return;
4481
4482         replaced = list_next_entry(fib6_entry, common.list);
4483
4484         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
4485         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
4486         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4487 }
4488
4489 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
4490                                     struct rt6_info *rt, bool replace)
4491 {
4492         struct mlxsw_sp_fib6_entry *fib6_entry;
4493         struct mlxsw_sp_fib_node *fib_node;
4494         int err;
4495
4496         if (mlxsw_sp->router->aborted)
4497                 return 0;
4498
4499         if (rt->rt6i_src.plen)
4500                 return -EINVAL;
4501
4502         if (mlxsw_sp_fib6_rt_should_ignore(rt))
4503                 return 0;
4504
4505         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
4506                                          &rt->rt6i_dst.addr,
4507                                          sizeof(rt->rt6i_dst.addr),
4508                                          rt->rt6i_dst.plen,
4509                                          MLXSW_SP_L3_PROTO_IPV6);
4510         if (IS_ERR(fib_node))
4511                 return PTR_ERR(fib_node);
4512
4513         /* Before creating a new entry, try to append route to an existing
4514          * multipath entry.
4515          */
4516         fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
4517         if (fib6_entry) {
4518                 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
4519                 if (err)
4520                         goto err_fib6_entry_nexthop_add;
4521                 return 0;
4522         }
4523
4524         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
4525         if (IS_ERR(fib6_entry)) {
4526                 err = PTR_ERR(fib6_entry);
4527                 goto err_fib6_entry_create;
4528         }
4529
4530         err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
4531         if (err)
4532                 goto err_fib6_node_entry_link;
4533
4534         mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
4535
4536         return 0;
4537
4538 err_fib6_node_entry_link:
4539         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4540 err_fib6_entry_create:
4541 err_fib6_entry_nexthop_add:
4542         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4543         return err;
4544 }
4545
4546 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
4547                                      struct rt6_info *rt)
4548 {
4549         struct mlxsw_sp_fib6_entry *fib6_entry;
4550         struct mlxsw_sp_fib_node *fib_node;
4551
4552         if (mlxsw_sp->router->aborted)
4553                 return;
4554
4555         if (mlxsw_sp_fib6_rt_should_ignore(rt))
4556                 return;
4557
4558         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
4559         if (WARN_ON(!fib6_entry))
4560                 return;
4561
4562         /* If route is part of a multipath entry, but not the last one
4563          * removed, then only reduce its nexthop group.
4564          */
4565         if (!list_is_singular(&fib6_entry->rt6_list)) {
4566                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
4567                 return;
4568         }
4569
4570         fib_node = fib6_entry->common.fib_node;
4571
4572         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4573         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4574         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4575 }
4576
4577 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
4578                                             enum mlxsw_reg_ralxx_protocol proto,
4579                                             u8 tree_id)
4580 {
4581         char ralta_pl[MLXSW_REG_RALTA_LEN];
4582         char ralst_pl[MLXSW_REG_RALST_LEN];
4583         int i, err;
4584
4585         mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
4586         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
4587         if (err)
4588                 return err;
4589
4590         mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
4591         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
4592         if (err)
4593                 return err;
4594
4595         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4596                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4597                 char raltb_pl[MLXSW_REG_RALTB_LEN];
4598                 char ralue_pl[MLXSW_REG_RALUE_LEN];
4599
4600                 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
4601                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
4602                                       raltb_pl);
4603                 if (err)
4604                         return err;
4605
4606                 mlxsw_reg_ralue_pack(ralue_pl, proto,
4607                                      MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
4608                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4609                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4610                                       ralue_pl);
4611                 if (err)
4612                         return err;
4613         }
4614
4615         return 0;
4616 }
4617
4618 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
4619 {
4620         enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
4621         int err;
4622
4623         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4624                                                MLXSW_SP_LPM_TREE_MIN);
4625         if (err)
4626                 return err;
4627
4628         proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
4629         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4630                                                 MLXSW_SP_LPM_TREE_MIN + 1);
4631 }
4632
4633 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
4634                                      struct mlxsw_sp_fib_node *fib_node)
4635 {
4636         struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
4637
4638         list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
4639                                  common.list) {
4640                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4641
4642                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4643                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4644                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4645                 /* Break when entry list is empty and node was freed.
4646                  * Otherwise, we'll access freed memory in the next
4647                  * iteration.
4648                  */
4649                 if (do_break)
4650                         break;
4651         }
4652 }
4653
4654 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
4655                                      struct mlxsw_sp_fib_node *fib_node)
4656 {
4657         struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
4658
4659         list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
4660                                  common.list) {
4661                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4662
4663                 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4664                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4665                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4666                 if (do_break)
4667                         break;
4668         }
4669 }
4670
4671 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
4672                                     struct mlxsw_sp_fib_node *fib_node)
4673 {
4674         switch (fib_node->fib->proto) {
4675         case MLXSW_SP_L3_PROTO_IPV4:
4676                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
4677                 break;
4678         case MLXSW_SP_L3_PROTO_IPV6:
4679                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
4680                 break;
4681         }
4682 }
4683
4684 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
4685                                   struct mlxsw_sp_vr *vr,
4686                                   enum mlxsw_sp_l3proto proto)
4687 {
4688         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
4689         struct mlxsw_sp_fib_node *fib_node, *tmp;
4690
4691         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
4692                 bool do_break = &tmp->list == &fib->node_list;
4693
4694                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
4695                 if (do_break)
4696                         break;
4697         }
4698 }
4699
4700 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
4701 {
4702         int i;
4703
4704         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4705                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4706
4707                 if (!mlxsw_sp_vr_is_used(vr))
4708                         continue;
4709                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
4710
4711                 /* If virtual router was only used for IPv4, then it's no
4712                  * longer used.
4713                  */
4714                 if (!mlxsw_sp_vr_is_used(vr))
4715                         continue;
4716                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
4717         }
4718 }
4719
4720 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
4721 {
4722         int err;
4723
4724         if (mlxsw_sp->router->aborted)
4725                 return;
4726         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
4727         mlxsw_sp_router_fib_flush(mlxsw_sp);
4728         mlxsw_sp->router->aborted = true;
4729         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
4730         if (err)
4731                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
4732 }
4733
4734 struct mlxsw_sp_fib_event_work {
4735         struct work_struct work;
4736         union {
4737                 struct fib6_entry_notifier_info fen6_info;
4738                 struct fib_entry_notifier_info fen_info;
4739                 struct fib_rule_notifier_info fr_info;
4740                 struct fib_nh_notifier_info fnh_info;
4741         };
4742         struct mlxsw_sp *mlxsw_sp;
4743         unsigned long event;
4744 };
4745
4746 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
4747 {
4748         struct mlxsw_sp_fib_event_work *fib_work =
4749                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4750         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4751         struct fib_rule *rule;
4752         bool replace, append;
4753         int err;
4754
4755         /* Protect internal structures from changes */
4756         rtnl_lock();
4757         switch (fib_work->event) {
4758         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4759         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4760         case FIB_EVENT_ENTRY_ADD:
4761                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4762                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
4763                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
4764                                                replace, append);
4765                 if (err)
4766                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4767                 fib_info_put(fib_work->fen_info.fi);
4768                 break;
4769         case FIB_EVENT_ENTRY_DEL:
4770                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
4771                 fib_info_put(fib_work->fen_info.fi);
4772                 break;
4773         case FIB_EVENT_RULE_ADD: /* fall through */
4774         case FIB_EVENT_RULE_DEL:
4775                 rule = fib_work->fr_info.rule;
4776                 if (!fib4_rule_default(rule) && !rule->l3mdev)
4777                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4778                 fib_rule_put(rule);
4779                 break;
4780         case FIB_EVENT_NH_ADD: /* fall through */
4781         case FIB_EVENT_NH_DEL:
4782                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
4783                                         fib_work->fnh_info.fib_nh);
4784                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
4785                 break;
4786         }
4787         rtnl_unlock();
4788         kfree(fib_work);
4789 }
4790
4791 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
4792 {
4793         struct mlxsw_sp_fib_event_work *fib_work =
4794                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4795         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4796         struct fib_rule *rule;
4797         bool replace;
4798         int err;
4799
4800         rtnl_lock();
4801         switch (fib_work->event) {
4802         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4803         case FIB_EVENT_ENTRY_ADD:
4804                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4805                 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
4806                                                fib_work->fen6_info.rt, replace);
4807                 if (err)
4808                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4809                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4810                 break;
4811         case FIB_EVENT_ENTRY_DEL:
4812                 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
4813                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4814                 break;
4815         case FIB_EVENT_RULE_ADD: /* fall through */
4816         case FIB_EVENT_RULE_DEL:
4817                 rule = fib_work->fr_info.rule;
4818                 if (!fib6_rule_default(rule) && !rule->l3mdev)
4819                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4820                 fib_rule_put(rule);
4821                 break;
4822         }
4823         rtnl_unlock();
4824         kfree(fib_work);
4825 }
4826
4827 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
4828                                        struct fib_notifier_info *info)
4829 {
4830         switch (fib_work->event) {
4831         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4832         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4833         case FIB_EVENT_ENTRY_ADD: /* fall through */
4834         case FIB_EVENT_ENTRY_DEL:
4835                 memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
4836                 /* Take referece on fib_info to prevent it from being
4837                  * freed while work is queued. Release it afterwards.
4838                  */
4839                 fib_info_hold(fib_work->fen_info.fi);
4840                 break;
4841         case FIB_EVENT_RULE_ADD: /* fall through */
4842         case FIB_EVENT_RULE_DEL:
4843                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4844                 fib_rule_get(fib_work->fr_info.rule);
4845                 break;
4846         case FIB_EVENT_NH_ADD: /* fall through */
4847         case FIB_EVENT_NH_DEL:
4848                 memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
4849                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
4850                 break;
4851         }
4852 }
4853
4854 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
4855                                        struct fib_notifier_info *info)
4856 {
4857         switch (fib_work->event) {
4858         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4859         case FIB_EVENT_ENTRY_ADD: /* fall through */
4860         case FIB_EVENT_ENTRY_DEL:
4861                 memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
4862                 rt6_hold(fib_work->fen6_info.rt);
4863                 break;
4864         case FIB_EVENT_RULE_ADD: /* fall through */
4865         case FIB_EVENT_RULE_DEL:
4866                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4867                 fib_rule_get(fib_work->fr_info.rule);
4868                 break;
4869         }
4870 }
4871
4872 /* Called with rcu_read_lock() */
4873 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
4874                                      unsigned long event, void *ptr)
4875 {
4876         struct mlxsw_sp_fib_event_work *fib_work;
4877         struct fib_notifier_info *info = ptr;
4878         struct mlxsw_sp_router *router;
4879
4880         if (!net_eq(info->net, &init_net) ||
4881             (info->family != AF_INET && info->family != AF_INET6))
4882                 return NOTIFY_DONE;
4883
4884         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
4885         if (WARN_ON(!fib_work))
4886                 return NOTIFY_BAD;
4887
4888         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
4889         fib_work->mlxsw_sp = router->mlxsw_sp;
4890         fib_work->event = event;
4891
4892         switch (info->family) {
4893         case AF_INET:
4894                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
4895                 mlxsw_sp_router_fib4_event(fib_work, info);
4896                 break;
4897         case AF_INET6:
4898                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
4899                 mlxsw_sp_router_fib6_event(fib_work, info);
4900                 break;
4901         }
4902
4903         mlxsw_core_schedule_work(&fib_work->work);
4904
4905         return NOTIFY_DONE;
4906 }
4907
4908 static struct mlxsw_sp_rif *
4909 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
4910                          const struct net_device *dev)
4911 {
4912         int i;
4913
4914         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
4915                 if (mlxsw_sp->router->rifs[i] &&
4916                     mlxsw_sp->router->rifs[i]->dev == dev)
4917                         return mlxsw_sp->router->rifs[i];
4918
4919         return NULL;
4920 }
4921
4922 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
4923 {
4924         char ritr_pl[MLXSW_REG_RITR_LEN];
4925         int err;
4926
4927         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
4928         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4929         if (WARN_ON_ONCE(err))
4930                 return err;
4931
4932         mlxsw_reg_ritr_enable_set(ritr_pl, false);
4933         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4934 }
4935
4936 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4937                                           struct mlxsw_sp_rif *rif)
4938 {
4939         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
4940         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
4941         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
4942 }
4943
4944 static bool
4945 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
4946                            unsigned long event)
4947 {
4948         struct inet6_dev *inet6_dev;
4949         bool addr_list_empty = true;
4950         struct in_device *idev;
4951
4952         switch (event) {
4953         case NETDEV_UP:
4954                 return rif == NULL;
4955         case NETDEV_DOWN:
4956                 idev = __in_dev_get_rtnl(dev);
4957                 if (idev && idev->ifa_list)
4958                         addr_list_empty = false;
4959
4960                 inet6_dev = __in6_dev_get(dev);
4961                 if (addr_list_empty && inet6_dev &&
4962                     !list_empty(&inet6_dev->addr_list))
4963                         addr_list_empty = false;
4964
4965                 if (rif && addr_list_empty &&
4966                     !netif_is_l3_slave(rif->dev))
4967                         return true;
4968                 /* It is possible we already removed the RIF ourselves
4969                  * if it was assigned to a netdev that is now a bridge
4970                  * or LAG slave.
4971                  */
4972                 return false;
4973         }
4974
4975         return false;
4976 }
4977
4978 static enum mlxsw_sp_rif_type
4979 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
4980                       const struct net_device *dev)
4981 {
4982         enum mlxsw_sp_fid_type type;
4983
4984         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
4985                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
4986
4987         /* Otherwise RIF type is derived from the type of the underlying FID. */
4988         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
4989                 type = MLXSW_SP_FID_TYPE_8021Q;
4990         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
4991                 type = MLXSW_SP_FID_TYPE_8021Q;
4992         else if (netif_is_bridge_master(dev))
4993                 type = MLXSW_SP_FID_TYPE_8021D;
4994         else
4995                 type = MLXSW_SP_FID_TYPE_RFID;
4996
4997         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
4998 }
4999
5000 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
5001 {
5002         int i;
5003
5004         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
5005                 if (!mlxsw_sp->router->rifs[i]) {
5006                         *p_rif_index = i;
5007                         return 0;
5008                 }
5009         }
5010
5011         return -ENOBUFS;
5012 }
5013
5014 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
5015                                                u16 vr_id,
5016                                                struct net_device *l3_dev)
5017 {
5018         struct mlxsw_sp_rif *rif;
5019
5020         rif = kzalloc(rif_size, GFP_KERNEL);
5021         if (!rif)
5022                 return NULL;
5023
5024         INIT_LIST_HEAD(&rif->nexthop_list);
5025         INIT_LIST_HEAD(&rif->neigh_list);
5026         ether_addr_copy(rif->addr, l3_dev->dev_addr);
5027         rif->mtu = l3_dev->mtu;
5028         rif->vr_id = vr_id;
5029         rif->dev = l3_dev;
5030         rif->rif_index = rif_index;
5031
5032         return rif;
5033 }
5034
5035 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
5036                                            u16 rif_index)
5037 {
5038         return mlxsw_sp->router->rifs[rif_index];
5039 }
5040
5041 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
5042 {
5043         return rif->rif_index;
5044 }
5045
5046 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
5047 {
5048         return lb_rif->common.rif_index;
5049 }
5050
5051 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
5052 {
5053         return lb_rif->ul_vr_id;
5054 }
5055
5056 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
5057 {
5058         return rif->dev->ifindex;
5059 }
5060
5061 static struct mlxsw_sp_rif *
5062 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
5063                     const struct mlxsw_sp_rif_params *params)
5064 {
5065         u32 tb_id = l3mdev_fib_table(params->dev);
5066         const struct mlxsw_sp_rif_ops *ops;
5067         struct mlxsw_sp_fid *fid = NULL;
5068         enum mlxsw_sp_rif_type type;
5069         struct mlxsw_sp_rif *rif;
5070         struct mlxsw_sp_vr *vr;
5071         u16 rif_index;
5072         int err;
5073
5074         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
5075         ops = mlxsw_sp->router->rif_ops_arr[type];
5076
5077         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
5078         if (IS_ERR(vr))
5079                 return ERR_CAST(vr);
5080         vr->rif_count++;
5081
5082         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
5083         if (err)
5084                 goto err_rif_index_alloc;
5085
5086         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
5087         if (!rif) {
5088                 err = -ENOMEM;
5089                 goto err_rif_alloc;
5090         }
5091         rif->mlxsw_sp = mlxsw_sp;
5092         rif->ops = ops;
5093
5094         if (ops->fid_get) {
5095                 fid = ops->fid_get(rif);
5096                 if (IS_ERR(fid)) {
5097                         err = PTR_ERR(fid);
5098                         goto err_fid_get;
5099                 }
5100                 rif->fid = fid;
5101         }
5102
5103         if (ops->setup)
5104                 ops->setup(rif, params);
5105
5106         err = ops->configure(rif);
5107         if (err)
5108                 goto err_configure;
5109
5110         mlxsw_sp_rif_counters_alloc(rif);
5111         mlxsw_sp->router->rifs[rif_index] = rif;
5112
5113         return rif;
5114
5115 err_configure:
5116         if (fid)
5117                 mlxsw_sp_fid_put(fid);
5118 err_fid_get:
5119         kfree(rif);
5120 err_rif_alloc:
5121 err_rif_index_alloc:
5122         vr->rif_count--;
5123         mlxsw_sp_vr_put(vr);
5124         return ERR_PTR(err);
5125 }
5126
5127 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
5128 {
5129         const struct mlxsw_sp_rif_ops *ops = rif->ops;
5130         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5131         struct mlxsw_sp_fid *fid = rif->fid;
5132         struct mlxsw_sp_vr *vr;
5133
5134         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
5135         vr = &mlxsw_sp->router->vrs[rif->vr_id];
5136
5137         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
5138         mlxsw_sp_rif_counters_free(rif);
5139         ops->deconfigure(rif);
5140         if (fid)
5141                 /* Loopback RIFs are not associated with a FID. */
5142                 mlxsw_sp_fid_put(fid);
5143         kfree(rif);
5144         vr->rif_count--;
5145         mlxsw_sp_vr_put(vr);
5146 }
5147
5148 static void
5149 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
5150                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
5151 {
5152         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5153
5154         params->vid = mlxsw_sp_port_vlan->vid;
5155         params->lag = mlxsw_sp_port->lagged;
5156         if (params->lag)
5157                 params->lag_id = mlxsw_sp_port->lag_id;
5158         else
5159                 params->system_port = mlxsw_sp_port->local_port;
5160 }
5161
5162 static int
5163 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
5164                                struct net_device *l3_dev)
5165 {
5166         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5167         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
5168         u16 vid = mlxsw_sp_port_vlan->vid;
5169         struct mlxsw_sp_rif *rif;
5170         struct mlxsw_sp_fid *fid;
5171         int err;
5172
5173         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5174         if (!rif) {
5175                 struct mlxsw_sp_rif_params params = {
5176                         .dev = l3_dev,
5177                 };
5178
5179                 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
5180                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
5181                 if (IS_ERR(rif))
5182                         return PTR_ERR(rif);
5183         }
5184
5185         /* FID was already created, just take a reference */
5186         fid = rif->ops->fid_get(rif);
5187         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
5188         if (err)
5189                 goto err_fid_port_vid_map;
5190
5191         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
5192         if (err)
5193                 goto err_port_vid_learning_set;
5194
5195         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
5196                                         BR_STATE_FORWARDING);
5197         if (err)
5198                 goto err_port_vid_stp_set;
5199
5200         mlxsw_sp_port_vlan->fid = fid;
5201
5202         return 0;
5203
5204 err_port_vid_stp_set:
5205         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
5206 err_port_vid_learning_set:
5207         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
5208 err_fid_port_vid_map:
5209         mlxsw_sp_fid_put(fid);
5210         return err;
5211 }
5212
5213 void
5214 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
5215 {
5216         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
5217         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
5218         u16 vid = mlxsw_sp_port_vlan->vid;
5219
5220         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
5221                 return;
5222
5223         mlxsw_sp_port_vlan->fid = NULL;
5224         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
5225         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
5226         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
5227         /* If router port holds the last reference on the rFID, then the
5228          * associated Sub-port RIF will be destroyed.
5229          */
5230         mlxsw_sp_fid_put(fid);
5231 }
5232
5233 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
5234                                              struct net_device *port_dev,
5235                                              unsigned long event, u16 vid)
5236 {
5237         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
5238         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
5239
5240         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
5241         if (WARN_ON(!mlxsw_sp_port_vlan))
5242                 return -EINVAL;
5243
5244         switch (event) {
5245         case NETDEV_UP:
5246                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
5247                                                       l3_dev);
5248         case NETDEV_DOWN:
5249                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
5250                 break;
5251         }
5252
5253         return 0;
5254 }
5255
5256 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
5257                                         unsigned long event)
5258 {
5259         if (netif_is_bridge_port(port_dev) ||
5260             netif_is_lag_port(port_dev) ||
5261             netif_is_ovs_port(port_dev))
5262                 return 0;
5263
5264         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
5265 }
5266
5267 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
5268                                          struct net_device *lag_dev,
5269                                          unsigned long event, u16 vid)
5270 {
5271         struct net_device *port_dev;
5272         struct list_head *iter;
5273         int err;
5274
5275         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
5276                 if (mlxsw_sp_port_dev_check(port_dev)) {
5277                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
5278                                                                 port_dev,
5279                                                                 event, vid);
5280                         if (err)
5281                                 return err;
5282                 }
5283         }
5284
5285         return 0;
5286 }
5287
5288 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
5289                                        unsigned long event)
5290 {
5291         if (netif_is_bridge_port(lag_dev))
5292                 return 0;
5293
5294         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
5295 }
5296
5297 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
5298                                           unsigned long event)
5299 {
5300         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5301         struct mlxsw_sp_rif_params params = {
5302                 .dev = l3_dev,
5303         };
5304         struct mlxsw_sp_rif *rif;
5305
5306         switch (event) {
5307         case NETDEV_UP:
5308                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
5309                 if (IS_ERR(rif))
5310                         return PTR_ERR(rif);
5311                 break;
5312         case NETDEV_DOWN:
5313                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5314                 mlxsw_sp_rif_destroy(rif);
5315                 break;
5316         }
5317
5318         return 0;
5319 }
5320
5321 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
5322                                         unsigned long event)
5323 {
5324         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
5325         u16 vid = vlan_dev_vlan_id(vlan_dev);
5326
5327         if (netif_is_bridge_port(vlan_dev))
5328                 return 0;
5329
5330         if (mlxsw_sp_port_dev_check(real_dev))
5331                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
5332                                                          event, vid);
5333         else if (netif_is_lag_master(real_dev))
5334                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
5335                                                      vid);
5336         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
5337                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
5338
5339         return 0;
5340 }
5341
5342 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
5343                                      unsigned long event)
5344 {
5345         if (mlxsw_sp_port_dev_check(dev))
5346                 return mlxsw_sp_inetaddr_port_event(dev, event);
5347         else if (netif_is_lag_master(dev))
5348                 return mlxsw_sp_inetaddr_lag_event(dev, event);
5349         else if (netif_is_bridge_master(dev))
5350                 return mlxsw_sp_inetaddr_bridge_event(dev, event);
5351         else if (is_vlan_dev(dev))
5352                 return mlxsw_sp_inetaddr_vlan_event(dev, event);
5353         else
5354                 return 0;
5355 }
5356
5357 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
5358                             unsigned long event, void *ptr)
5359 {
5360         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
5361         struct net_device *dev = ifa->ifa_dev->dev;
5362         struct mlxsw_sp *mlxsw_sp;
5363         struct mlxsw_sp_rif *rif;
5364         int err = 0;
5365
5366         mlxsw_sp = mlxsw_sp_lower_get(dev);
5367         if (!mlxsw_sp)
5368                 goto out;
5369
5370         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5371         if (!mlxsw_sp_rif_should_config(rif, dev, event))
5372                 goto out;
5373
5374         err = __mlxsw_sp_inetaddr_event(dev, event);
5375 out:
5376         return notifier_from_errno(err);
5377 }
5378
5379 struct mlxsw_sp_inet6addr_event_work {
5380         struct work_struct work;
5381         struct net_device *dev;
5382         unsigned long event;
5383 };
5384
5385 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
5386 {
5387         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
5388                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
5389         struct net_device *dev = inet6addr_work->dev;
5390         unsigned long event = inet6addr_work->event;
5391         struct mlxsw_sp *mlxsw_sp;
5392         struct mlxsw_sp_rif *rif;
5393
5394         rtnl_lock();
5395         mlxsw_sp = mlxsw_sp_lower_get(dev);
5396         if (!mlxsw_sp)
5397                 goto out;
5398
5399         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5400         if (!mlxsw_sp_rif_should_config(rif, dev, event))
5401                 goto out;
5402
5403         __mlxsw_sp_inetaddr_event(dev, event);
5404 out:
5405         rtnl_unlock();
5406         dev_put(dev);
5407         kfree(inet6addr_work);
5408 }
5409
5410 /* Called with rcu_read_lock() */
5411 int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
5412                              unsigned long event, void *ptr)
5413 {
5414         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
5415         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
5416         struct net_device *dev = if6->idev->dev;
5417
5418         if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
5419                 return NOTIFY_DONE;
5420
5421         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
5422         if (!inet6addr_work)
5423                 return NOTIFY_BAD;
5424
5425         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
5426         inet6addr_work->dev = dev;
5427         inet6addr_work->event = event;
5428         dev_hold(dev);
5429         mlxsw_core_schedule_work(&inet6addr_work->work);
5430
5431         return NOTIFY_DONE;
5432 }
5433
5434 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
5435                              const char *mac, int mtu)
5436 {
5437         char ritr_pl[MLXSW_REG_RITR_LEN];
5438         int err;
5439
5440         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
5441         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5442         if (err)
5443                 return err;
5444
5445         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
5446         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
5447         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
5448         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5449 }
5450
5451 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
5452 {
5453         struct mlxsw_sp *mlxsw_sp;
5454         struct mlxsw_sp_rif *rif;
5455         u16 fid_index;
5456         int err;
5457
5458         mlxsw_sp = mlxsw_sp_lower_get(dev);
5459         if (!mlxsw_sp)
5460                 return 0;
5461
5462         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5463         if (!rif)
5464                 return 0;
5465         fid_index = mlxsw_sp_fid_index(rif->fid);
5466
5467         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
5468         if (err)
5469                 return err;
5470
5471         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
5472                                 dev->mtu);
5473         if (err)
5474                 goto err_rif_edit;
5475
5476         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
5477         if (err)
5478                 goto err_rif_fdb_op;
5479
5480         ether_addr_copy(rif->addr, dev->dev_addr);
5481         rif->mtu = dev->mtu;
5482
5483         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
5484
5485         return 0;
5486
5487 err_rif_fdb_op:
5488         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
5489 err_rif_edit:
5490         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
5491         return err;
5492 }
5493
5494 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
5495                                   struct net_device *l3_dev)
5496 {
5497         struct mlxsw_sp_rif *rif;
5498
5499         /* If netdev is already associated with a RIF, then we need to
5500          * destroy it and create a new one with the new virtual router ID.
5501          */
5502         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5503         if (rif)
5504                 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5505
5506         return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
5507 }
5508
5509 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
5510                                     struct net_device *l3_dev)
5511 {
5512         struct mlxsw_sp_rif *rif;
5513
5514         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5515         if (!rif)
5516                 return;
5517         __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5518 }
5519
5520 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
5521                                  struct netdev_notifier_changeupper_info *info)
5522 {
5523         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5524         int err = 0;
5525
5526         if (!mlxsw_sp)
5527                 return 0;
5528
5529         switch (event) {
5530         case NETDEV_PRECHANGEUPPER:
5531                 return 0;
5532         case NETDEV_CHANGEUPPER:
5533                 if (info->linking)
5534                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
5535                 else
5536                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
5537                 break;
5538         }
5539
5540         return err;
5541 }
5542
5543 static struct mlxsw_sp_rif_subport *
5544 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
5545 {
5546         return container_of(rif, struct mlxsw_sp_rif_subport, common);
5547 }
5548
5549 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
5550                                        const struct mlxsw_sp_rif_params *params)
5551 {
5552         struct mlxsw_sp_rif_subport *rif_subport;
5553
5554         rif_subport = mlxsw_sp_rif_subport_rif(rif);
5555         rif_subport->vid = params->vid;
5556         rif_subport->lag = params->lag;
5557         if (params->lag)
5558                 rif_subport->lag_id = params->lag_id;
5559         else
5560                 rif_subport->system_port = params->system_port;
5561 }
5562
5563 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
5564 {
5565         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5566         struct mlxsw_sp_rif_subport *rif_subport;
5567         char ritr_pl[MLXSW_REG_RITR_LEN];
5568
5569         rif_subport = mlxsw_sp_rif_subport_rif(rif);
5570         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
5571                             rif->rif_index, rif->vr_id, rif->dev->mtu);
5572         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5573         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
5574                                   rif_subport->lag ? rif_subport->lag_id :
5575                                                      rif_subport->system_port,
5576                                   rif_subport->vid);
5577
5578         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5579 }
5580
5581 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
5582 {
5583         int err;
5584
5585         err = mlxsw_sp_rif_subport_op(rif, true);
5586         if (err)
5587                 return err;
5588
5589         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5590                                   mlxsw_sp_fid_index(rif->fid), true);
5591         if (err)
5592                 goto err_rif_fdb_op;
5593
5594         mlxsw_sp_fid_rif_set(rif->fid, rif);
5595         return 0;
5596
5597 err_rif_fdb_op:
5598         mlxsw_sp_rif_subport_op(rif, false);
5599         return err;
5600 }
5601
5602 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
5603 {
5604         struct mlxsw_sp_fid *fid = rif->fid;
5605
5606         mlxsw_sp_fid_rif_set(fid, NULL);
5607         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5608                             mlxsw_sp_fid_index(fid), false);
5609         mlxsw_sp_rif_subport_op(rif, false);
5610 }
5611
5612 static struct mlxsw_sp_fid *
5613 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
5614 {
5615         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
5616 }
5617
5618 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
5619         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
5620         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
5621         .setup                  = mlxsw_sp_rif_subport_setup,
5622         .configure              = mlxsw_sp_rif_subport_configure,
5623         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
5624         .fid_get                = mlxsw_sp_rif_subport_fid_get,
5625 };
5626
5627 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
5628                                     enum mlxsw_reg_ritr_if_type type,
5629                                     u16 vid_fid, bool enable)
5630 {
5631         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5632         char ritr_pl[MLXSW_REG_RITR_LEN];
5633
5634         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
5635                             rif->dev->mtu);
5636         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5637         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
5638
5639         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5640 }
5641
5642 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
5643 {
5644         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
5645 }
5646
5647 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
5648 {
5649         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5650         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5651         int err;
5652
5653         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
5654         if (err)
5655                 return err;
5656
5657         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5658                                      mlxsw_sp_router_port(mlxsw_sp), true);
5659         if (err)
5660                 goto err_fid_mc_flood_set;
5661
5662         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5663                                      mlxsw_sp_router_port(mlxsw_sp), true);
5664         if (err)
5665                 goto err_fid_bc_flood_set;
5666
5667         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5668                                   mlxsw_sp_fid_index(rif->fid), true);
5669         if (err)
5670                 goto err_rif_fdb_op;
5671
5672         mlxsw_sp_fid_rif_set(rif->fid, rif);
5673         return 0;
5674
5675 err_rif_fdb_op:
5676         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5677                                mlxsw_sp_router_port(mlxsw_sp), false);
5678 err_fid_bc_flood_set:
5679         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5680                                mlxsw_sp_router_port(mlxsw_sp), false);
5681 err_fid_mc_flood_set:
5682         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5683         return err;
5684 }
5685
5686 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
5687 {
5688         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5689         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5690         struct mlxsw_sp_fid *fid = rif->fid;
5691
5692         mlxsw_sp_fid_rif_set(fid, NULL);
5693         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5694                             mlxsw_sp_fid_index(fid), false);
5695         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5696                                mlxsw_sp_router_port(mlxsw_sp), false);
5697         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5698                                mlxsw_sp_router_port(mlxsw_sp), false);
5699         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5700 }
5701
5702 static struct mlxsw_sp_fid *
5703 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
5704 {
5705         u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
5706
5707         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
5708 }
5709
5710 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
5711         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
5712         .rif_size               = sizeof(struct mlxsw_sp_rif),
5713         .configure              = mlxsw_sp_rif_vlan_configure,
5714         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
5715         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
5716 };
5717
5718 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
5719 {
5720         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5721         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5722         int err;
5723
5724         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
5725                                        true);
5726         if (err)
5727                 return err;
5728
5729         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5730                                      mlxsw_sp_router_port(mlxsw_sp), true);
5731         if (err)
5732                 goto err_fid_mc_flood_set;
5733
5734         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5735                                      mlxsw_sp_router_port(mlxsw_sp), true);
5736         if (err)
5737                 goto err_fid_bc_flood_set;
5738
5739         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5740                                   mlxsw_sp_fid_index(rif->fid), true);
5741         if (err)
5742                 goto err_rif_fdb_op;
5743
5744         mlxsw_sp_fid_rif_set(rif->fid, rif);
5745         return 0;
5746
5747 err_rif_fdb_op:
5748         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5749                                mlxsw_sp_router_port(mlxsw_sp), false);
5750 err_fid_bc_flood_set:
5751         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5752                                mlxsw_sp_router_port(mlxsw_sp), false);
5753 err_fid_mc_flood_set:
5754         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5755         return err;
5756 }
5757
5758 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
5759 {
5760         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5761         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5762         struct mlxsw_sp_fid *fid = rif->fid;
5763
5764         mlxsw_sp_fid_rif_set(fid, NULL);
5765         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5766                             mlxsw_sp_fid_index(fid), false);
5767         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5768                                mlxsw_sp_router_port(mlxsw_sp), false);
5769         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5770                                mlxsw_sp_router_port(mlxsw_sp), false);
5771         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5772 }
5773
5774 static struct mlxsw_sp_fid *
5775 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
5776 {
5777         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
5778 }
5779
5780 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
5781         .type                   = MLXSW_SP_RIF_TYPE_FID,
5782         .rif_size               = sizeof(struct mlxsw_sp_rif),
5783         .configure              = mlxsw_sp_rif_fid_configure,
5784         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
5785         .fid_get                = mlxsw_sp_rif_fid_fid_get,
5786 };
5787
5788 static struct mlxsw_sp_rif_ipip_lb *
5789 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
5790 {
5791         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
5792 }
5793
5794 static void
5795 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
5796                            const struct mlxsw_sp_rif_params *params)
5797 {
5798         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
5799         struct mlxsw_sp_rif_ipip_lb *rif_lb;
5800
5801         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
5802                                  common);
5803         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
5804         rif_lb->lb_config = params_lb->lb_config;
5805 }
5806
5807 static int
5808 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
5809                         struct mlxsw_sp_vr *ul_vr, bool enable)
5810 {
5811         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
5812         struct mlxsw_sp_rif *rif = &lb_rif->common;
5813         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5814         char ritr_pl[MLXSW_REG_RITR_LEN];
5815         u32 saddr4;
5816
5817         switch (lb_cf.ul_protocol) {
5818         case MLXSW_SP_L3_PROTO_IPV4:
5819                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
5820                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
5821                                     rif->rif_index, rif->vr_id, rif->dev->mtu);
5822                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
5823                             MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
5824                             ul_vr->id, saddr4, lb_cf.okey);
5825                 break;
5826
5827         case MLXSW_SP_L3_PROTO_IPV6:
5828                 return -EAFNOSUPPORT;
5829         }
5830
5831         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5832 }
5833
5834 static int
5835 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
5836 {
5837         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5838         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
5839         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5840         struct mlxsw_sp_vr *ul_vr;
5841         int err;
5842
5843         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id);
5844         if (IS_ERR(ul_vr))
5845                 return PTR_ERR(ul_vr);
5846
5847         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
5848         if (err)
5849                 goto err_loopback_op;
5850
5851         lb_rif->ul_vr_id = ul_vr->id;
5852         ++ul_vr->rif_count;
5853         return 0;
5854
5855 err_loopback_op:
5856         mlxsw_sp_vr_put(ul_vr);
5857         return err;
5858 }
5859
5860 static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
5861 {
5862         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5863         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5864         struct mlxsw_sp_vr *ul_vr;
5865
5866         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
5867         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
5868
5869         --ul_vr->rif_count;
5870         mlxsw_sp_vr_put(ul_vr);
5871 }
5872
5873 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
5874         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
5875         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
5876         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
5877         .configure              = mlxsw_sp_rif_ipip_lb_configure,
5878         .deconfigure            = mlxsw_sp_rif_ipip_lb_deconfigure,
5879 };
5880
5881 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
5882         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
5883         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_ops,
5884         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
5885         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp_rif_ipip_lb_ops,
5886 };
5887
5888 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
5889 {
5890         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5891
5892         mlxsw_sp->router->rifs = kcalloc(max_rifs,
5893                                          sizeof(struct mlxsw_sp_rif *),
5894                                          GFP_KERNEL);
5895         if (!mlxsw_sp->router->rifs)
5896                 return -ENOMEM;
5897
5898         mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
5899
5900         return 0;
5901 }
5902
5903 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
5904 {
5905         int i;
5906
5907         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5908                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
5909
5910         kfree(mlxsw_sp->router->rifs);
5911 }
5912
5913 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
5914 {
5915         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
5916         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
5917         return 0;
5918 }
5919
5920 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
5921 {
5922         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
5923 }
5924
5925 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
5926 {
5927         struct mlxsw_sp_router *router;
5928
5929         /* Flush pending FIB notifications and then flush the device's
5930          * table before requesting another dump. The FIB notification
5931          * block is unregistered, so no need to take RTNL.
5932          */
5933         mlxsw_core_flush_owq();
5934         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
5935         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
5936 }
5937
5938 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5939 {
5940         char rgcr_pl[MLXSW_REG_RGCR_LEN];
5941         u64 max_rifs;
5942         int err;
5943
5944         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
5945                 return -EIO;
5946         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5947
5948         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
5949         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
5950         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5951         if (err)
5952                 return err;
5953         return 0;
5954 }
5955
5956 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5957 {
5958         char rgcr_pl[MLXSW_REG_RGCR_LEN];
5959
5960         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
5961         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5962 }
5963
5964 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5965 {
5966         struct mlxsw_sp_router *router;
5967         int err;
5968
5969         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
5970         if (!router)
5971                 return -ENOMEM;
5972         mlxsw_sp->router = router;
5973         router->mlxsw_sp = mlxsw_sp;
5974
5975         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
5976         err = __mlxsw_sp_router_init(mlxsw_sp);
5977         if (err)
5978                 goto err_router_init;
5979
5980         err = mlxsw_sp_rifs_init(mlxsw_sp);
5981         if (err)
5982                 goto err_rifs_init;
5983
5984         err = mlxsw_sp_ipips_init(mlxsw_sp);
5985         if (err)
5986                 goto err_ipips_init;
5987
5988         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
5989                               &mlxsw_sp_nexthop_ht_params);
5990         if (err)
5991                 goto err_nexthop_ht_init;
5992
5993         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
5994                               &mlxsw_sp_nexthop_group_ht_params);
5995         if (err)
5996                 goto err_nexthop_group_ht_init;
5997
5998         err = mlxsw_sp_lpm_init(mlxsw_sp);
5999         if (err)
6000                 goto err_lpm_init;
6001
6002         err = mlxsw_sp_vrs_init(mlxsw_sp);
6003         if (err)
6004                 goto err_vrs_init;
6005
6006         err = mlxsw_sp_neigh_init(mlxsw_sp);
6007         if (err)
6008                 goto err_neigh_init;
6009
6010         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
6011         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
6012                                     mlxsw_sp_router_fib_dump_flush);
6013         if (err)
6014                 goto err_register_fib_notifier;
6015
6016         return 0;
6017
6018 err_register_fib_notifier:
6019         mlxsw_sp_neigh_fini(mlxsw_sp);
6020 err_neigh_init:
6021         mlxsw_sp_vrs_fini(mlxsw_sp);
6022 err_vrs_init:
6023         mlxsw_sp_lpm_fini(mlxsw_sp);
6024 err_lpm_init:
6025         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
6026 err_nexthop_group_ht_init:
6027         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
6028 err_nexthop_ht_init:
6029         mlxsw_sp_ipips_fini(mlxsw_sp);
6030 err_ipips_init:
6031         mlxsw_sp_rifs_fini(mlxsw_sp);
6032 err_rifs_init:
6033         __mlxsw_sp_router_fini(mlxsw_sp);
6034 err_router_init:
6035         kfree(mlxsw_sp->router);
6036         return err;
6037 }
6038
6039 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
6040 {
6041         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
6042         mlxsw_sp_neigh_fini(mlxsw_sp);
6043         mlxsw_sp_vrs_fini(mlxsw_sp);
6044         mlxsw_sp_lpm_fini(mlxsw_sp);
6045         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
6046         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
6047         mlxsw_sp_ipips_fini(mlxsw_sp);
6048         mlxsw_sp_rifs_fini(mlxsw_sp);
6049         __mlxsw_sp_router_fini(mlxsw_sp);
6050         kfree(mlxsw_sp->router);
6051 }