drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <linux/inetdevice.h>
  44 #include <linux/netdevice.h>
  45 #include <linux/if_bridge.h>
  46 #include <net/netevent.h>
  47 #include <net/neighbour.h>
  48 #include <net/arp.h>
  49 #include <net/ip_fib.h>
  50 #include <net/fib_rules.h>
  51 #include <net/l3mdev.h>
  52
  53 #include "spectrum.h"
  54 #include "core.h"
  55 #include "reg.h"
  56 #include "spectrum_cnt.h"
  57 #include "spectrum_dpipe.h"
  58 #include "spectrum_router.h"
  59
  60 struct mlxsw_sp_vr;
  61 struct mlxsw_sp_lpm_tree;
  62 struct mlxsw_sp_rif_ops;
  63
  64 struct mlxsw_sp_router {
  65         struct mlxsw_sp *mlxsw_sp;
  66         struct mlxsw_sp_rif **rifs;
  67         struct mlxsw_sp_vr *vrs;
  68         struct rhashtable neigh_ht;
  69         struct rhashtable nexthop_group_ht;
  70         struct rhashtable nexthop_ht;
  71         struct {
  72                 struct mlxsw_sp_lpm_tree *trees;
  73                 unsigned int tree_count;
  74         } lpm;
  75         struct {
  76                 struct delayed_work dw;
  77                 unsigned long interval; /* ms */
  78         } neighs_update;
  79         struct delayed_work nexthop_probe_dw;
  80 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  81         struct list_head nexthop_neighs_list;
  82         bool aborted;
  83         struct notifier_block fib_nb;
  84         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  85 };
  86
  87 struct mlxsw_sp_rif {
  88         struct list_head nexthop_list;
  89         struct list_head neigh_list;
  90         struct net_device *dev;
  91         struct mlxsw_sp_fid *fid;
  92         unsigned char addr[ETH_ALEN];
  93         int mtu;
  94         u16 rif_index;
  95         u16 vr_id;
  96         const struct mlxsw_sp_rif_ops *ops;
  97         struct mlxsw_sp *mlxsw_sp;
  98
  99         unsigned int counter_ingress;
 100         bool counter_ingress_valid;
 101         unsigned int counter_egress;
 102         bool counter_egress_valid;
 103 };
 104
 105 struct mlxsw_sp_rif_params {
 106         struct net_device *dev;
 107         union {
 108                 u16 system_port;
 109                 u16 lag_id;
 110         };
 111         u16 vid;
 112         bool lag;
 113 };
 114
 115 struct mlxsw_sp_rif_subport {
 116         struct mlxsw_sp_rif common;
 117         union {
 118                 u16 system_port;
 119                 u16 lag_id;
 120         };
 121         u16 vid;
 122         bool lag;
 123 };
 124
 125 struct mlxsw_sp_rif_ops {
 126         enum mlxsw_sp_rif_type type;
 127         size_t rif_size;
 128
 129         void (*setup)(struct mlxsw_sp_rif *rif,
 130                       const struct mlxsw_sp_rif_params *params);
 131         int (*configure)(struct mlxsw_sp_rif *rif);
 132         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 133         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 134 };
 135
 136 static unsigned int *
 137 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 138                            enum mlxsw_sp_rif_counter_dir dir)
 139 {
 140         switch (dir) {
 141         case MLXSW_SP_RIF_COUNTER_EGRESS:
 142                 return &rif->counter_egress;
 143         case MLXSW_SP_RIF_COUNTER_INGRESS:
 144                 return &rif->counter_ingress;
 145         }
 146         return NULL;
 147 }
 148
 149 static bool
 150 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 151                                enum mlxsw_sp_rif_counter_dir dir)
 152 {
 153         switch (dir) {
 154         case MLXSW_SP_RIF_COUNTER_EGRESS:
 155                 return rif->counter_egress_valid;
 156         case MLXSW_SP_RIF_COUNTER_INGRESS:
 157                 return rif->counter_ingress_valid;
 158         }
 159         return false;
 160 }
 161
 162 static void
 163 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 164                                enum mlxsw_sp_rif_counter_dir dir,
 165                                bool valid)
 166 {
 167         switch (dir) {
 168         case MLXSW_SP_RIF_COUNTER_EGRESS:
 169                 rif->counter_egress_valid = valid;
 170                 break;
 171         case MLXSW_SP_RIF_COUNTER_INGRESS:
 172                 rif->counter_ingress_valid = valid;
 173                 break;
 174         }
 175 }
 176
 177 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 178                                      unsigned int counter_index, bool enable,
 179                                      enum mlxsw_sp_rif_counter_dir dir)
 180 {
 181         char ritr_pl[MLXSW_REG_RITR_LEN];
 182         bool is_egress = false;
 183         int err;
 184
 185         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 186                 is_egress = true;
 187         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 188         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 189         if (err)
 190                 return err;
 191
 192         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 193                                     is_egress);
 194         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 195 }
 196
 197 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 198                                    struct mlxsw_sp_rif *rif,
 199                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 200 {
 201         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 202         unsigned int *p_counter_index;
 203         bool valid;
 204         int err;
 205
 206         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 207         if (!valid)
 208                 return -EINVAL;
 209
 210         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 211         if (!p_counter_index)
 212                 return -EINVAL;
 213         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 214                              MLXSW_REG_RICNT_OPCODE_NOP);
 215         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 216         if (err)
 217                 return err;
 218         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 219         return 0;
 220 }
 221
 222 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 223                                       unsigned int counter_index)
 224 {
 225         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 226
 227         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 228                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 229         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 230 }
 231
 232 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 233                                struct mlxsw_sp_rif *rif,
 234                                enum mlxsw_sp_rif_counter_dir dir)
 235 {
 236         unsigned int *p_counter_index;
 237         int err;
 238
 239         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 240         if (!p_counter_index)
 241                 return -EINVAL;
 242         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 243                                      p_counter_index);
 244         if (err)
 245                 return err;
 246
 247         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 248         if (err)
 249                 goto err_counter_clear;
 250
 251         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 252                                         *p_counter_index, true, dir);
 253         if (err)
 254                 goto err_counter_edit;
 255         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 256         return 0;
 257
 258 err_counter_edit:
 259 err_counter_clear:
 260         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 261                               *p_counter_index);
 262         return err;
 263 }
 264
 265 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 266                                struct mlxsw_sp_rif *rif,
 267                                enum mlxsw_sp_rif_counter_dir dir)
 268 {
 269         unsigned int *p_counter_index;
 270
 271         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 272                 return;
 273
 274         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 275         if (WARN_ON(!p_counter_index))
 276                 return;
 277         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 278                                   *p_counter_index, false, dir);
 279         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 280                               *p_counter_index);
 281         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 282 }
 283
 284 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 285 {
 286         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 287         struct devlink *devlink;
 288
 289         devlink = priv_to_devlink(mlxsw_sp->core);
 290         if (!devlink_dpipe_table_counter_enabled(devlink,
 291                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 292                 return;
 293         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 294 }
 295
 296 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 297 {
 298         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 299
 300         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 301 }
 302
 303 static struct mlxsw_sp_rif *
 304 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 305                          const struct net_device *dev);
 306
 307 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE)
 308
 309 struct mlxsw_sp_prefix_usage {
 310         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 311 };
 312
 313 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 314         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 315
 316 static bool
 317 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
 318                              struct mlxsw_sp_prefix_usage *prefix_usage2)
 319 {
 320         unsigned char prefix;
 321
 322         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
 323                 if (!test_bit(prefix, prefix_usage2->b))
 324                         return false;
 325         }
 326         return true;
 327 }
 328
 329 static bool
 330 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 331                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 332 {
 333         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 334 }
 335
 336 static bool
 337 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
 338 {
 339         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
 340
 341         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 342 }
 343
 344 static void
 345 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 346                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 347 {
 348         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 349 }
 350
 351 static void
 352 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 353                           unsigned char prefix_len)
 354 {
 355         set_bit(prefix_len, prefix_usage->b);
 356 }
 357
 358 static void
 359 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 360                             unsigned char prefix_len)
 361 {
 362         clear_bit(prefix_len, prefix_usage->b);
 363 }
 364
 365 struct mlxsw_sp_fib_key {
 366         unsigned char addr[sizeof(struct in6_addr)];
 367         unsigned char prefix_len;
 368 };
 369
 370 enum mlxsw_sp_fib_entry_type {
 371         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 372         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 373         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 374 };
 375
 376 struct mlxsw_sp_nexthop_group;
 377 struct mlxsw_sp_fib;
 378
 379 struct mlxsw_sp_fib_node {
 380         struct list_head entry_list;
 381         struct list_head list;
 382         struct rhash_head ht_node;
 383         struct mlxsw_sp_fib *fib;
 384         struct mlxsw_sp_fib_key key;
 385 };
 386
 387 struct mlxsw_sp_fib_entry_params {
 388         u32 tb_id;
 389         u32 prio;
 390         u8 tos;
 391         u8 type;
 392 };
 393
 394 struct mlxsw_sp_fib_entry {
 395         struct list_head list;
 396         struct mlxsw_sp_fib_node *fib_node;
 397         enum mlxsw_sp_fib_entry_type type;
 398         struct list_head nexthop_group_node;
 399         struct mlxsw_sp_nexthop_group *nh_group;
 400         struct mlxsw_sp_fib_entry_params params;
 401         bool offloaded;
 402 };
 403
 404 enum mlxsw_sp_l3proto {
 405         MLXSW_SP_L3_PROTO_IPV4,
 406         MLXSW_SP_L3_PROTO_IPV6,
 407 };
 408
 409 struct mlxsw_sp_lpm_tree {
 410         u8 id; /* tree ID */
 411         unsigned int ref_count;
 412         enum mlxsw_sp_l3proto proto;
 413         struct mlxsw_sp_prefix_usage prefix_usage;
 414 };
 415
 416 struct mlxsw_sp_fib {
 417         struct rhashtable ht;
 418         struct list_head node_list;
 419         struct mlxsw_sp_vr *vr;
 420         struct mlxsw_sp_lpm_tree *lpm_tree;
 421         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 422         struct mlxsw_sp_prefix_usage prefix_usage;
 423         enum mlxsw_sp_l3proto proto;
 424 };
 425
 426 struct mlxsw_sp_vr {
 427         u16 id; /* virtual router ID */
 428         u32 tb_id; /* kernel fib table id */
 429         unsigned int rif_count;
 430         struct mlxsw_sp_fib *fib4;
 431 };
 432
 433 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 434
 435 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 436                                                 enum mlxsw_sp_l3proto proto)
 437 {
 438         struct mlxsw_sp_fib *fib;
 439         int err;
 440
 441         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 442         if (!fib)
 443                 return ERR_PTR(-ENOMEM);
 444         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 445         if (err)
 446                 goto err_rhashtable_init;
 447         INIT_LIST_HEAD(&fib->node_list);
 448         fib->proto = proto;
 449         fib->vr = vr;
 450         return fib;
 451
 452 err_rhashtable_init:
 453         kfree(fib);
 454         return ERR_PTR(err);
 455 }
 456
 457 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 458 {
 459         WARN_ON(!list_empty(&fib->node_list));
 460         WARN_ON(fib->lpm_tree);
 461         rhashtable_destroy(&fib->ht);
 462         kfree(fib);
 463 }
 464
 465 static struct mlxsw_sp_lpm_tree *
 466 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 467 {
 468         static struct mlxsw_sp_lpm_tree *lpm_tree;
 469         int i;
 470
 471         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 472                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 473                 if (lpm_tree->ref_count == 0)
 474                         return lpm_tree;
 475         }
 476         return NULL;
 477 }
 478
 479 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 480                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 481 {
 482         char ralta_pl[MLXSW_REG_RALTA_LEN];
 483
 484         mlxsw_reg_ralta_pack(ralta_pl, true,
 485                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 486                              lpm_tree->id);
 487         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 488 }
 489
 490 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 491                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 492 {
 493         char ralta_pl[MLXSW_REG_RALTA_LEN];
 494
 495         mlxsw_reg_ralta_pack(ralta_pl, false,
 496                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 497                              lpm_tree->id);
 498         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 499 }
 500
 501 static int
 502 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 503                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 504                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 505 {
 506         char ralst_pl[MLXSW_REG_RALST_LEN];
 507         u8 root_bin = 0;
 508         u8 prefix;
 509         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 510
 511         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 512                 root_bin = prefix;
 513
 514         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 515         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 516                 if (prefix == 0)
 517                         continue;
 518                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 519                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 520                 last_prefix = prefix;
 521         }
 522         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 523 }
 524
 525 static struct mlxsw_sp_lpm_tree *
 526 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 527                          struct mlxsw_sp_prefix_usage *prefix_usage,
 528                          enum mlxsw_sp_l3proto proto)
 529 {
 530         struct mlxsw_sp_lpm_tree *lpm_tree;
 531         int err;
 532
 533         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 534         if (!lpm_tree)
 535                 return ERR_PTR(-EBUSY);
 536         lpm_tree->proto = proto;
 537         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 538         if (err)
 539                 return ERR_PTR(err);
 540
 541         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 542                                                 lpm_tree);
 543         if (err)
 544                 goto err_left_struct_set;
 545         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 546                sizeof(lpm_tree->prefix_usage));
 547         return lpm_tree;
 548
 549 err_left_struct_set:
 550         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 551         return ERR_PTR(err);
 552 }
 553
 554 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 555                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 556 {
 557         return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 558 }
 559
 560 static struct mlxsw_sp_lpm_tree *
 561 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 562                       struct mlxsw_sp_prefix_usage *prefix_usage,
 563                       enum mlxsw_sp_l3proto proto)
 564 {
 565         struct mlxsw_sp_lpm_tree *lpm_tree;
 566         int i;
 567
 568         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 569                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 570                 if (lpm_tree->ref_count != 0 &&
 571                     lpm_tree->proto == proto &&
 572                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 573                                              prefix_usage))
 574                         goto inc_ref_count;
 575         }
 576         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 577                                             proto);
 578         if (IS_ERR(lpm_tree))
 579                 return lpm_tree;
 580
 581 inc_ref_count:
 582         lpm_tree->ref_count++;
 583         return lpm_tree;
 584 }
 585
 586 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 587                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 588 {
 589         if (--lpm_tree->ref_count == 0)
 590                 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 591         return 0;
 592 }
 593
 594 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 595
 596 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 597 {
 598         struct mlxsw_sp_lpm_tree *lpm_tree;
 599         u64 max_trees;
 600         int i;
 601
 602         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 603                 return -EIO;
 604
 605         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 606         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 607         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 608                                              sizeof(struct mlxsw_sp_lpm_tree),
 609                                              GFP_KERNEL);
 610         if (!mlxsw_sp->router->lpm.trees)
 611                 return -ENOMEM;
 612
 613         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 614                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 615                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 616         }
 617
 618         return 0;
 619 }
 620
 621 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 622 {
 623         kfree(mlxsw_sp->router->lpm.trees);
 624 }
 625
 626 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 627 {
 628         return !!vr->fib4;
 629 }
 630
 631 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 632 {
 633         struct mlxsw_sp_vr *vr;
 634         int i;
 635
 636         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 637                 vr = &mlxsw_sp->router->vrs[i];
 638                 if (!mlxsw_sp_vr_is_used(vr))
 639                         return vr;
 640         }
 641         return NULL;
 642 }
 643
 644 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 645                                      const struct mlxsw_sp_fib *fib)
 646 {
 647         char raltb_pl[MLXSW_REG_RALTB_LEN];
 648
 649         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 650                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 651                              fib->lpm_tree->id);
 652         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 653 }
 654
 655 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 656                                        const struct mlxsw_sp_fib *fib)
 657 {
 658         char raltb_pl[MLXSW_REG_RALTB_LEN];
 659
 660         /* Bind to tree 0 which is default */
 661         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 662                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 663         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 664 }
 665
 666 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 667 {
 668         /* For our purpose, squash main and local table into one */
 669         if (tb_id == RT_TABLE_LOCAL)
 670                 tb_id = RT_TABLE_MAIN;
 671         return tb_id;
 672 }
 673
 674 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 675                                             u32 tb_id)
 676 {
 677         struct mlxsw_sp_vr *vr;
 678         int i;
 679
 680         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 681
 682         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 683                 vr = &mlxsw_sp->router->vrs[i];
 684                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 685                         return vr;
 686         }
 687         return NULL;
 688 }
 689
 690 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 691                                             enum mlxsw_sp_l3proto proto)
 692 {
 693         switch (proto) {
 694         case MLXSW_SP_L3_PROTO_IPV4:
 695                 return vr->fib4;
 696         case MLXSW_SP_L3_PROTO_IPV6:
 697                 BUG_ON(1);
 698         }
 699         return NULL;
 700 }
 701
 702 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 703                                               u32 tb_id)
 704 {
 705         struct mlxsw_sp_vr *vr;
 706
 707         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 708         if (!vr)
 709                 return ERR_PTR(-EBUSY);
 710         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 711         if (IS_ERR(vr->fib4))
 712                 return ERR_CAST(vr->fib4);
 713         vr->tb_id = tb_id;
 714         return vr;
 715 }
 716
 717 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 718 {
 719         mlxsw_sp_fib_destroy(vr->fib4);
 720         vr->fib4 = NULL;
 721 }
 722
 723 static int
 724 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
 725                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 726 {
 727         struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
 728         struct mlxsw_sp_lpm_tree *new_tree;
 729         int err;
 730
 731         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
 732                 return 0;
 733
 734         new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 735                                          fib->proto);
 736         if (IS_ERR(new_tree)) {
 737                 /* We failed to get a tree according to the required
 738                  * prefix usage. However, the current tree might be still good
 739                  * for us if our requirement is subset of the prefixes used
 740                  * in the tree.
 741                  */
 742                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 743                                                  &lpm_tree->prefix_usage))
 744                         return 0;
 745                 return PTR_ERR(new_tree);
 746         }
 747
 748         /* Prevent packet loss by overwriting existing binding */
 749         fib->lpm_tree = new_tree;
 750         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
 751         if (err)
 752                 goto err_tree_bind;
 753         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 754
 755         return 0;
 756
 757 err_tree_bind:
 758         fib->lpm_tree = lpm_tree;
 759         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 760         return err;
 761 }
 762
 763 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 764 {
 765         struct mlxsw_sp_vr *vr;
 766
 767         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 768         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 769         if (!vr)
 770                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 771         return vr;
 772 }
 773
 774 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 775 {
 776         if (!vr->rif_count && list_empty(&vr->fib4->node_list))
 777                 mlxsw_sp_vr_destroy(vr);
 778 }
 779
 780 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 781 {
 782         struct mlxsw_sp_vr *vr;
 783         u64 max_vrs;
 784         int i;
 785
 786         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 787                 return -EIO;
 788
 789         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 790         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 791                                         GFP_KERNEL);
 792         if (!mlxsw_sp->router->vrs)
 793                 return -ENOMEM;
 794
 795         for (i = 0; i < max_vrs; i++) {
 796                 vr = &mlxsw_sp->router->vrs[i];
 797                 vr->id = i;
 798         }
 799
 800         return 0;
 801 }
 802
 803 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 804
 805 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 806 {
 807         /* At this stage we're guaranteed not to have new incoming
 808          * FIB notifications and the work queue is free from FIBs
 809          * sitting on top of mlxsw netdevs. However, we can still
 810          * have other FIBs queued. Flush the queue before flushing
 811          * the device's tables. No need for locks, as we're the only
 812          * writer.
 813          */
 814         mlxsw_core_flush_owq();
 815         mlxsw_sp_router_fib_flush(mlxsw_sp);
 816         kfree(mlxsw_sp->router->vrs);
 817 }
 818
 819 struct mlxsw_sp_neigh_key {
 820         struct neighbour *n;
 821 };
 822
 823 struct mlxsw_sp_neigh_entry {
 824         struct list_head rif_list_node;
 825         struct rhash_head ht_node;
 826         struct mlxsw_sp_neigh_key key;
 827         u16 rif;
 828         bool connected;
 829         unsigned char ha[ETH_ALEN];
 830         struct list_head nexthop_list; /* list of nexthops using
 831                                         * this neigh entry
 832                                         */
 833         struct list_head nexthop_neighs_list_node;
 834 };
 835
 836 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 837         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 838         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 839         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 840 };
 841
 842 static struct mlxsw_sp_neigh_entry *
 843 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 844                            u16 rif)
 845 {
 846         struct mlxsw_sp_neigh_entry *neigh_entry;
 847
 848         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
 849         if (!neigh_entry)
 850                 return NULL;
 851
 852         neigh_entry->key.n = n;
 853         neigh_entry->rif = rif;
 854         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 855
 856         return neigh_entry;
 857 }
 858
 859 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 860 {
 861         kfree(neigh_entry);
 862 }
 863
 864 static int
 865 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 866                             struct mlxsw_sp_neigh_entry *neigh_entry)
 867 {
 868         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
 869                                       &neigh_entry->ht_node,
 870                                       mlxsw_sp_neigh_ht_params);
 871 }
 872
 873 static void
 874 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 875                             struct mlxsw_sp_neigh_entry *neigh_entry)
 876 {
 877         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
 878                                &neigh_entry->ht_node,
 879                                mlxsw_sp_neigh_ht_params);
 880 }
 881
 882 static struct mlxsw_sp_neigh_entry *
 883 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 884 {
 885         struct mlxsw_sp_neigh_entry *neigh_entry;
 886         struct mlxsw_sp_rif *rif;
 887         int err;
 888
 889         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 890         if (!rif)
 891                 return ERR_PTR(-EINVAL);
 892
 893         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
 894         if (!neigh_entry)
 895                 return ERR_PTR(-ENOMEM);
 896
 897         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 898         if (err)
 899                 goto err_neigh_entry_insert;
 900
 901         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
 902
 903         return neigh_entry;
 904
 905 err_neigh_entry_insert:
 906         mlxsw_sp_neigh_entry_free(neigh_entry);
 907         return ERR_PTR(err);
 908 }
 909
 910 static void
 911 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 912                              struct mlxsw_sp_neigh_entry *neigh_entry)
 913 {
 914         list_del(&neigh_entry->rif_list_node);
 915         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 916         mlxsw_sp_neigh_entry_free(neigh_entry);
 917 }
 918
 919 static struct mlxsw_sp_neigh_entry *
 920 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 921 {
 922         struct mlxsw_sp_neigh_key key;
 923
 924         key.n = n;
 925         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
 926                                       &key, mlxsw_sp_neigh_ht_params);
 927 }
 928
 929 static void
 930 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 931 {
 932         unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 933
 934         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
 935 }
 936
 937 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 938                                                    char *rauhtd_pl,
 939                                                    int ent_index)
 940 {
 941         struct net_device *dev;
 942         struct neighbour *n;
 943         __be32 dipn;
 944         u32 dip;
 945         u16 rif;
 946
 947         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 948
 949         if (!mlxsw_sp->router->rifs[rif]) {
 950                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 951                 return;
 952         }
 953
 954         dipn = htonl(dip);
 955         dev = mlxsw_sp->router->rifs[rif]->dev;
 956         n = neigh_lookup(&arp_tbl, &dipn, dev);
 957         if (!n) {
 958                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
 959                            &dip);
 960                 return;
 961         }
 962
 963         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
 964         neigh_event_send(n, NULL);
 965         neigh_release(n);
 966 }
 967
 968 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 969                                                    char *rauhtd_pl,
 970                                                    int rec_index)
 971 {
 972         u8 num_entries;
 973         int i;
 974
 975         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 976                                                                 rec_index);
 977         /* Hardware starts counting at 0, so add 1. */
 978         num_entries++;
 979
 980         /* Each record consists of several neighbour entries. */
 981         for (i = 0; i < num_entries; i++) {
 982                 int ent_index;
 983
 984                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
 985                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
 986                                                        ent_index);
 987         }
 988
 989 }
 990
 991 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 992                                               char *rauhtd_pl, int rec_index)
 993 {
 994         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
 995         case MLXSW_REG_RAUHTD_TYPE_IPV4:
 996                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
 997                                                        rec_index);
 998                 break;
 999         case MLXSW_REG_RAUHTD_TYPE_IPV6:
1000                 WARN_ON_ONCE(1);
1001                 break;
1002         }
1003 }
1004
1005 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1006 {
1007         u8 num_rec, last_rec_index, num_entries;
1008
1009         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1010         last_rec_index = num_rec - 1;
1011
1012         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1013                 return false;
1014         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1015             MLXSW_REG_RAUHTD_TYPE_IPV6)
1016                 return true;
1017
1018         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1019                                                                 last_rec_index);
1020         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1021                 return true;
1022         return false;
1023 }
1024
1025 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1026 {
1027         char *rauhtd_pl;
1028         u8 num_rec;
1029         int i, err;
1030
1031         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1032         if (!rauhtd_pl)
1033                 return -ENOMEM;
1034
1035         /* Make sure the neighbour's netdev isn't removed in the
1036          * process.
1037          */
1038         rtnl_lock();
1039         do {
1040                 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
1041                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1042                                       rauhtd_pl);
1043                 if (err) {
1044                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1045                         break;
1046                 }
1047                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1048                 for (i = 0; i < num_rec; i++)
1049                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1050                                                           i);
1051         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1052         rtnl_unlock();
1053
1054         kfree(rauhtd_pl);
1055         return err;
1056 }
1057
1058 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1059 {
1060         struct mlxsw_sp_neigh_entry *neigh_entry;
1061
1062         /* Take RTNL mutex here to prevent lists from changes */
1063         rtnl_lock();
1064         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1065                             nexthop_neighs_list_node)
1066                 /* If this neigh have nexthops, make the kernel think this neigh
1067                  * is active regardless of the traffic.
1068                  */
1069                 neigh_event_send(neigh_entry->key.n, NULL);
1070         rtnl_unlock();
1071 }
1072
1073 static void
1074 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1075 {
1076         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1077
1078         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1079                                msecs_to_jiffies(interval));
1080 }
1081
1082 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1083 {
1084         struct mlxsw_sp_router *router;
1085         int err;
1086
1087         router = container_of(work, struct mlxsw_sp_router,
1088                               neighs_update.dw.work);
1089         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1090         if (err)
1091                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1092
1093         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1094
1095         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1096 }
1097
1098 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1099 {
1100         struct mlxsw_sp_neigh_entry *neigh_entry;
1101         struct mlxsw_sp_router *router;
1102
1103         router = container_of(work, struct mlxsw_sp_router,
1104                               nexthop_probe_dw.work);
1105         /* Iterate over nexthop neighbours, find those who are unresolved and
1106          * send arp on them. This solves the chicken-egg problem when
1107          * the nexthop wouldn't get offloaded until the neighbor is resolved
1108          * but it wouldn't get resolved ever in case traffic is flowing in HW
1109          * using different nexthop.
1110          *
1111          * Take RTNL mutex here to prevent lists from changes.
1112          */
1113         rtnl_lock();
1114         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1115                             nexthop_neighs_list_node)
1116                 if (!neigh_entry->connected)
1117                         neigh_event_send(neigh_entry->key.n, NULL);
1118         rtnl_unlock();
1119
1120         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1121                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1122 }
1123
1124 static void
1125 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1126                               struct mlxsw_sp_neigh_entry *neigh_entry,
1127                               bool removing);
1128
1129 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1130 {
1131         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1132                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1133 }
1134
1135 static void
1136 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1137                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1138                                 enum mlxsw_reg_rauht_op op)
1139 {
1140         struct neighbour *n = neigh_entry->key.n;
1141         u32 dip = ntohl(*((__be32 *) n->primary_key));
1142         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1143
1144         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1145                               dip);
1146         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1147 }
1148
1149 static void
1150 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1151                             struct mlxsw_sp_neigh_entry *neigh_entry,
1152                             bool adding)
1153 {
1154         if (!adding && !neigh_entry->connected)
1155                 return;
1156         neigh_entry->connected = adding;
1157         if (neigh_entry->key.n->tbl == &arp_tbl)
1158                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1159                                                 mlxsw_sp_rauht_op(adding));
1160         else
1161                 WARN_ON_ONCE(1);
1162 }
1163
1164 struct mlxsw_sp_neigh_event_work {
1165         struct work_struct work;
1166         struct mlxsw_sp *mlxsw_sp;
1167         struct neighbour *n;
1168 };
1169
1170 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1171 {
1172         struct mlxsw_sp_neigh_event_work *neigh_work =
1173                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1174         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1175         struct mlxsw_sp_neigh_entry *neigh_entry;
1176         struct neighbour *n = neigh_work->n;
1177         unsigned char ha[ETH_ALEN];
1178         bool entry_connected;
1179         u8 nud_state, dead;
1180
1181         /* If these parameters are changed after we release the lock,
1182          * then we are guaranteed to receive another event letting us
1183          * know about it.
1184          */
1185         read_lock_bh(&n->lock);
1186         memcpy(ha, n->ha, ETH_ALEN);
1187         nud_state = n->nud_state;
1188         dead = n->dead;
1189         read_unlock_bh(&n->lock);
1190
1191         rtnl_lock();
1192         entry_connected = nud_state & NUD_VALID && !dead;
1193         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1194         if (!entry_connected && !neigh_entry)
1195                 goto out;
1196         if (!neigh_entry) {
1197                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1198                 if (IS_ERR(neigh_entry))
1199                         goto out;
1200         }
1201
1202         memcpy(neigh_entry->ha, ha, ETH_ALEN);
1203         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1204         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1205
1206         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1207                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1208
1209 out:
1210         rtnl_unlock();
1211         neigh_release(n);
1212         kfree(neigh_work);
1213 }
1214
1215 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1216                                    unsigned long event, void *ptr)
1217 {
1218         struct mlxsw_sp_neigh_event_work *neigh_work;
1219         struct mlxsw_sp_port *mlxsw_sp_port;
1220         struct mlxsw_sp *mlxsw_sp;
1221         unsigned long interval;
1222         struct neigh_parms *p;
1223         struct neighbour *n;
1224
1225         switch (event) {
1226         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1227                 p = ptr;
1228
1229                 /* We don't care about changes in the default table. */
1230                 if (!p->dev || p->tbl != &arp_tbl)
1231                         return NOTIFY_DONE;
1232
1233                 /* We are in atomic context and can't take RTNL mutex,
1234                  * so use RCU variant to walk the device chain.
1235                  */
1236                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1237                 if (!mlxsw_sp_port)
1238                         return NOTIFY_DONE;
1239
1240                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1241                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1242                 mlxsw_sp->router->neighs_update.interval = interval;
1243
1244                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1245                 break;
1246         case NETEVENT_NEIGH_UPDATE:
1247                 n = ptr;
1248
1249                 if (n->tbl != &arp_tbl)
1250                         return NOTIFY_DONE;
1251
1252                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1253                 if (!mlxsw_sp_port)
1254                         return NOTIFY_DONE;
1255
1256                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1257                 if (!neigh_work) {
1258                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1259                         return NOTIFY_BAD;
1260                 }
1261
1262                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1263                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1264                 neigh_work->n = n;
1265
1266                 /* Take a reference to ensure the neighbour won't be
1267                  * destructed until we drop the reference in delayed
1268                  * work.
1269                  */
1270                 neigh_clone(n);
1271                 mlxsw_core_schedule_work(&neigh_work->work);
1272                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1273                 break;
1274         }
1275
1276         return NOTIFY_DONE;
1277 }
1278
1279 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1280 {
1281         int err;
1282
1283         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1284                               &mlxsw_sp_neigh_ht_params);
1285         if (err)
1286                 return err;
1287
1288         /* Initialize the polling interval according to the default
1289          * table.
1290          */
1291         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1292
1293         /* Create the delayed works for the activity_update */
1294         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1295                           mlxsw_sp_router_neighs_update_work);
1296         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1297                           mlxsw_sp_router_probe_unresolved_nexthops);
1298         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1299         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1300         return 0;
1301 }
1302
1303 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1304 {
1305         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1306         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1307         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
1308 }
1309
1310 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1311                                     const struct mlxsw_sp_rif *rif)
1312 {
1313         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1314
1315         mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1316                              rif->rif_index, rif->addr);
1317         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1318 }
1319
1320 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1321                                          struct mlxsw_sp_rif *rif)
1322 {
1323         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1324
1325         mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif);
1326         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1327                                  rif_list_node)
1328                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1329 }
1330
1331 struct mlxsw_sp_nexthop_key {
1332         struct fib_nh *fib_nh;
1333 };
1334
1335 struct mlxsw_sp_nexthop {
1336         struct list_head neigh_list_node; /* member of neigh entry list */
1337         struct list_head rif_list_node;
1338         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1339                                                 * this belongs to
1340                                                 */
1341         struct rhash_head ht_node;
1342         struct mlxsw_sp_nexthop_key key;
1343         struct mlxsw_sp_rif *rif;
1344         u8 should_offload:1, /* set indicates this neigh is connected and
1345                               * should be put to KVD linear area of this group.
1346                               */
1347            offloaded:1, /* set in case the neigh is actually put into
1348                          * KVD linear area of this group.
1349                          */
1350            update:1; /* set indicates that MAC of this neigh should be
1351                       * updated in HW
1352                       */
1353         struct mlxsw_sp_neigh_entry *neigh_entry;
1354 };
1355
1356 struct mlxsw_sp_nexthop_group_key {
1357         struct fib_info *fi;
1358 };
1359
1360 struct mlxsw_sp_nexthop_group {
1361         struct rhash_head ht_node;
1362         struct list_head fib_list; /* list of fib entries that use this group */
1363         struct mlxsw_sp_nexthop_group_key key;
1364         u8 adj_index_valid:1,
1365            gateway:1; /* routes using the group use a gateway */
1366         u32 adj_index;
1367         u16 ecmp_size;
1368         u16 count;
1369         struct mlxsw_sp_nexthop nexthops[0];
1370 #define nh_rif  nexthops[0].rif
1371 };
1372
1373 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1374         .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1375         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1376         .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1377 };
1378
1379 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1380                                          struct mlxsw_sp_nexthop_group *nh_grp)
1381 {
1382         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
1383                                       &nh_grp->ht_node,
1384                                       mlxsw_sp_nexthop_group_ht_params);
1385 }
1386
1387 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1388                                           struct mlxsw_sp_nexthop_group *nh_grp)
1389 {
1390         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
1391                                &nh_grp->ht_node,
1392                                mlxsw_sp_nexthop_group_ht_params);
1393 }
1394
1395 static struct mlxsw_sp_nexthop_group *
1396 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1397                               struct mlxsw_sp_nexthop_group_key key)
1398 {
1399         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, &key,
1400                                       mlxsw_sp_nexthop_group_ht_params);
1401 }
1402
1403 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1404         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1405         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1406         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1407 };
1408
1409 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1410                                    struct mlxsw_sp_nexthop *nh)
1411 {
1412         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
1413                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1414 }
1415
1416 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1417                                     struct mlxsw_sp_nexthop *nh)
1418 {
1419         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
1420                                mlxsw_sp_nexthop_ht_params);
1421 }
1422
1423 static struct mlxsw_sp_nexthop *
1424 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1425                         struct mlxsw_sp_nexthop_key key)
1426 {
1427         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
1428                                       mlxsw_sp_nexthop_ht_params);
1429 }
1430
1431 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1432                                              const struct mlxsw_sp_fib *fib,
1433                                              u32 adj_index, u16 ecmp_size,
1434                                              u32 new_adj_index,
1435                                              u16 new_ecmp_size)
1436 {
1437         char raleu_pl[MLXSW_REG_RALEU_LEN];
1438
1439         mlxsw_reg_raleu_pack(raleu_pl,
1440                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
1441                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
1442                              new_ecmp_size);
1443         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1444 }
1445
1446 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1447                                           struct mlxsw_sp_nexthop_group *nh_grp,
1448                                           u32 old_adj_index, u16 old_ecmp_size)
1449 {
1450         struct mlxsw_sp_fib_entry *fib_entry;
1451         struct mlxsw_sp_fib *fib = NULL;
1452         int err;
1453
1454         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1455                 if (fib == fib_entry->fib_node->fib)
1456                         continue;
1457                 fib = fib_entry->fib_node->fib;
1458                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1459                                                         old_adj_index,
1460                                                         old_ecmp_size,
1461                                                         nh_grp->adj_index,
1462                                                         nh_grp->ecmp_size);
1463                 if (err)
1464                         return err;
1465         }
1466         return 0;
1467 }
1468
1469 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1470                                        struct mlxsw_sp_nexthop *nh)
1471 {
1472         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1473         char ratr_pl[MLXSW_REG_RATR_LEN];
1474
1475         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1476                             true, adj_index, neigh_entry->rif);
1477         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1478         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1479 }
1480
1481 static int
1482 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1483                                   struct mlxsw_sp_nexthop_group *nh_grp,
1484                                   bool reallocate)
1485 {
1486         u32 adj_index = nh_grp->adj_index; /* base */
1487         struct mlxsw_sp_nexthop *nh;
1488         int i;
1489         int err;
1490
1491         for (i = 0; i < nh_grp->count; i++) {
1492                 nh = &nh_grp->nexthops[i];
1493
1494                 if (!nh->should_offload) {
1495                         nh->offloaded = 0;
1496                         continue;
1497                 }
1498
1499                 if (nh->update || reallocate) {
1500                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1501                                                           adj_index, nh);
1502                         if (err)
1503                                 return err;
1504                         nh->update = 0;
1505                         nh->offloaded = 1;
1506                 }
1507                 adj_index++;
1508         }
1509         return 0;
1510 }
1511
1512 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1513                                      struct mlxsw_sp_fib_entry *fib_entry);
1514
1515 static int
1516 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1517                                     struct mlxsw_sp_nexthop_group *nh_grp)
1518 {
1519         struct mlxsw_sp_fib_entry *fib_entry;
1520         int err;
1521
1522         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1523                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1524                 if (err)
1525                         return err;
1526         }
1527         return 0;
1528 }
1529
1530 static void
1531 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1532                                struct mlxsw_sp_nexthop_group *nh_grp)
1533 {
1534         struct mlxsw_sp_nexthop *nh;
1535         bool offload_change = false;
1536         u32 adj_index;
1537         u16 ecmp_size = 0;
1538         bool old_adj_index_valid;
1539         u32 old_adj_index;
1540         u16 old_ecmp_size;
1541         int i;
1542         int err;
1543
1544         if (!nh_grp->gateway) {
1545                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1546                 return;
1547         }
1548
1549         for (i = 0; i < nh_grp->count; i++) {
1550                 nh = &nh_grp->nexthops[i];
1551
1552                 if (nh->should_offload ^ nh->offloaded) {
1553                         offload_change = true;
1554                         if (nh->should_offload)
1555                                 nh->update = 1;
1556                 }
1557                 if (nh->should_offload)
1558                         ecmp_size++;
1559         }
1560         if (!offload_change) {
1561                 /* Nothing was added or removed, so no need to reallocate. Just
1562                  * update MAC on existing adjacency indexes.
1563                  */
1564                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1565                                                         false);
1566                 if (err) {
1567                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1568                         goto set_trap;
1569                 }
1570                 return;
1571         }
1572         if (!ecmp_size)
1573                 /* No neigh of this group is connected so we just set
1574                  * the trap and let everthing flow through kernel.
1575                  */
1576                 goto set_trap;
1577
1578         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
1579         if (err) {
1580                 /* We ran out of KVD linear space, just set the
1581                  * trap and let everything flow through kernel.
1582                  */
1583                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1584                 goto set_trap;
1585         }
1586         old_adj_index_valid = nh_grp->adj_index_valid;
1587         old_adj_index = nh_grp->adj_index;
1588         old_ecmp_size = nh_grp->ecmp_size;
1589         nh_grp->adj_index_valid = 1;
1590         nh_grp->adj_index = adj_index;
1591         nh_grp->ecmp_size = ecmp_size;
1592         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1593         if (err) {
1594                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1595                 goto set_trap;
1596         }
1597
1598         if (!old_adj_index_valid) {
1599                 /* The trap was set for fib entries, so we have to call
1600                  * fib entry update to unset it and use adjacency index.
1601                  */
1602                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1603                 if (err) {
1604                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1605                         goto set_trap;
1606                 }
1607                 return;
1608         }
1609
1610         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1611                                              old_adj_index, old_ecmp_size);
1612         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1613         if (err) {
1614                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1615                 goto set_trap;
1616         }
1617         return;
1618
1619 set_trap:
1620         old_adj_index_valid = nh_grp->adj_index_valid;
1621         nh_grp->adj_index_valid = 0;
1622         for (i = 0; i < nh_grp->count; i++) {
1623                 nh = &nh_grp->nexthops[i];
1624                 nh->offloaded = 0;
1625         }
1626         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1627         if (err)
1628                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1629         if (old_adj_index_valid)
1630                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1631 }
1632
1633 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1634                                             bool removing)
1635 {
1636         if (!removing && !nh->should_offload)
1637                 nh->should_offload = 1;
1638         else if (removing && nh->offloaded)
1639                 nh->should_offload = 0;
1640         nh->update = 1;
1641 }
1642
1643 static void
1644 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1645                               struct mlxsw_sp_neigh_entry *neigh_entry,
1646                               bool removing)
1647 {
1648         struct mlxsw_sp_nexthop *nh;
1649
1650         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1651                             neigh_list_node) {
1652                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1653                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1654         }
1655 }
1656
1657 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1658                                       struct mlxsw_sp_rif *rif)
1659 {
1660         if (nh->rif)
1661                 return;
1662
1663         nh->rif = rif;
1664         list_add(&nh->rif_list_node, &rif->nexthop_list);
1665 }
1666
1667 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1668 {
1669         if (!nh->rif)
1670                 return;
1671
1672         list_del(&nh->rif_list_node);
1673         nh->rif = NULL;
1674 }
1675
1676 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1677                                        struct mlxsw_sp_nexthop *nh)
1678 {
1679         struct mlxsw_sp_neigh_entry *neigh_entry;
1680         struct fib_nh *fib_nh = nh->key.fib_nh;
1681         struct neighbour *n;
1682         u8 nud_state, dead;
1683         int err;
1684
1685         if (!nh->nh_grp->gateway || nh->neigh_entry)
1686                 return 0;
1687
1688         /* Take a reference of neigh here ensuring that neigh would
1689          * not be detructed before the nexthop entry is finished.
1690          * The reference is taken either in neigh_lookup() or
1691          * in neigh_create() in case n is not found.
1692          */
1693         n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1694         if (!n) {
1695                 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1696                 if (IS_ERR(n))
1697                         return PTR_ERR(n);
1698                 neigh_event_send(n, NULL);
1699         }
1700         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1701         if (!neigh_entry) {
1702                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1703                 if (IS_ERR(neigh_entry)) {
1704                         err = -EINVAL;
1705                         goto err_neigh_entry_create;
1706                 }
1707         }
1708
1709         /* If that is the first nexthop connected to that neigh, add to
1710          * nexthop_neighs_list
1711          */
1712         if (list_empty(&neigh_entry->nexthop_list))
1713                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1714                               &mlxsw_sp->router->nexthop_neighs_list);
1715
1716         nh->neigh_entry = neigh_entry;
1717         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1718         read_lock_bh(&n->lock);
1719         nud_state = n->nud_state;
1720         dead = n->dead;
1721         read_unlock_bh(&n->lock);
1722         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1723
1724         return 0;
1725
1726 err_neigh_entry_create:
1727         neigh_release(n);
1728         return err;
1729 }
1730
1731 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1732                                         struct mlxsw_sp_nexthop *nh)
1733 {
1734         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1735         struct neighbour *n;
1736
1737         if (!neigh_entry)
1738                 return;
1739         n = neigh_entry->key.n;
1740
1741         __mlxsw_sp_nexthop_neigh_update(nh, true);
1742         list_del(&nh->neigh_list_node);
1743         nh->neigh_entry = NULL;
1744
1745         /* If that is the last nexthop connected to that neigh, remove from
1746          * nexthop_neighs_list
1747          */
1748         if (list_empty(&neigh_entry->nexthop_list))
1749                 list_del(&neigh_entry->nexthop_neighs_list_node);
1750
1751         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1752                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1753
1754         neigh_release(n);
1755 }
1756
1757 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1758                                  struct mlxsw_sp_nexthop_group *nh_grp,
1759                                  struct mlxsw_sp_nexthop *nh,
1760                                  struct fib_nh *fib_nh)
1761 {
1762         struct net_device *dev = fib_nh->nh_dev;
1763         struct in_device *in_dev;
1764         struct mlxsw_sp_rif *rif;
1765         int err;
1766
1767         nh->nh_grp = nh_grp;
1768         nh->key.fib_nh = fib_nh;
1769         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1770         if (err)
1771                 return err;
1772
1773         if (!dev)
1774                 return 0;
1775
1776         in_dev = __in_dev_get_rtnl(dev);
1777         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1778             fib_nh->nh_flags & RTNH_F_LINKDOWN)
1779                 return 0;
1780
1781         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1782         if (!rif)
1783                 return 0;
1784         mlxsw_sp_nexthop_rif_init(nh, rif);
1785
1786         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1787         if (err)
1788                 goto err_nexthop_neigh_init;
1789
1790         return 0;
1791
1792 err_nexthop_neigh_init:
1793         mlxsw_sp_nexthop_rif_fini(nh);
1794         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1795         return err;
1796 }
1797
1798 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1799                                   struct mlxsw_sp_nexthop *nh)
1800 {
1801         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1802         mlxsw_sp_nexthop_rif_fini(nh);
1803         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1804 }
1805
1806 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1807                                    unsigned long event, struct fib_nh *fib_nh)
1808 {
1809         struct mlxsw_sp_nexthop_key key;
1810         struct mlxsw_sp_nexthop *nh;
1811         struct mlxsw_sp_rif *rif;
1812
1813         if (mlxsw_sp->router->aborted)
1814                 return;
1815
1816         key.fib_nh = fib_nh;
1817         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1818         if (WARN_ON_ONCE(!nh))
1819                 return;
1820
1821         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1822         if (!rif)
1823                 return;
1824
1825         switch (event) {
1826         case FIB_EVENT_NH_ADD:
1827                 mlxsw_sp_nexthop_rif_init(nh, rif);
1828                 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1829                 break;
1830         case FIB_EVENT_NH_DEL:
1831                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1832                 mlxsw_sp_nexthop_rif_fini(nh);
1833                 break;
1834         }
1835
1836         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1837 }
1838
1839 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1840                                            struct mlxsw_sp_rif *rif)
1841 {
1842         struct mlxsw_sp_nexthop *nh, *tmp;
1843
1844         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
1845                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1846                 mlxsw_sp_nexthop_rif_fini(nh);
1847                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1848         }
1849 }
1850
1851 static struct mlxsw_sp_nexthop_group *
1852 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1853 {
1854         struct mlxsw_sp_nexthop_group *nh_grp;
1855         struct mlxsw_sp_nexthop *nh;
1856         struct fib_nh *fib_nh;
1857         size_t alloc_size;
1858         int i;
1859         int err;
1860
1861         alloc_size = sizeof(*nh_grp) +
1862                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1863         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1864         if (!nh_grp)
1865                 return ERR_PTR(-ENOMEM);
1866         INIT_LIST_HEAD(&nh_grp->fib_list);
1867         nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1868         nh_grp->count = fi->fib_nhs;
1869         nh_grp->key.fi = fi;
1870         for (i = 0; i < nh_grp->count; i++) {
1871                 nh = &nh_grp->nexthops[i];
1872                 fib_nh = &fi->fib_nh[i];
1873                 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1874                 if (err)
1875                         goto err_nexthop_init;
1876         }
1877         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1878         if (err)
1879                 goto err_nexthop_group_insert;
1880         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1881         return nh_grp;
1882
1883 err_nexthop_group_insert:
1884 err_nexthop_init:
1885         for (i--; i >= 0; i--) {
1886                 nh = &nh_grp->nexthops[i];
1887                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1888         }
1889         kfree(nh_grp);
1890         return ERR_PTR(err);
1891 }
1892
1893 static void
1894 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1895                                struct mlxsw_sp_nexthop_group *nh_grp)
1896 {
1897         struct mlxsw_sp_nexthop *nh;
1898         int i;
1899
1900         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1901         for (i = 0; i < nh_grp->count; i++) {
1902                 nh = &nh_grp->nexthops[i];
1903                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1904         }
1905         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1906         WARN_ON_ONCE(nh_grp->adj_index_valid);
1907         kfree(nh_grp);
1908 }
1909
1910 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1911                                       struct mlxsw_sp_fib_entry *fib_entry,
1912                                       struct fib_info *fi)
1913 {
1914         struct mlxsw_sp_nexthop_group_key key;
1915         struct mlxsw_sp_nexthop_group *nh_grp;
1916
1917         key.fi = fi;
1918         nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1919         if (!nh_grp) {
1920                 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1921                 if (IS_ERR(nh_grp))
1922                         return PTR_ERR(nh_grp);
1923         }
1924         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1925         fib_entry->nh_group = nh_grp;
1926         return 0;
1927 }
1928
1929 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1930                                        struct mlxsw_sp_fib_entry *fib_entry)
1931 {
1932         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1933
1934         list_del(&fib_entry->nexthop_group_node);
1935         if (!list_empty(&nh_grp->fib_list))
1936                 return;
1937         mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1938 }
1939
1940 static bool
1941 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1942 {
1943         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1944
1945         if (fib_entry->params.tos)
1946                 return false;
1947
1948         switch (fib_entry->type) {
1949         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1950                 return !!nh_group->adj_index_valid;
1951         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1952                 return !!nh_group->nh_rif;
1953         default:
1954                 return false;
1955         }
1956 }
1957
1958 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1959 {
1960         fib_entry->offloaded = true;
1961
1962         switch (fib_entry->fib_node->fib->proto) {
1963         case MLXSW_SP_L3_PROTO_IPV4:
1964                 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1965                 break;
1966         case MLXSW_SP_L3_PROTO_IPV6:
1967                 WARN_ON_ONCE(1);
1968         }
1969 }
1970
1971 static void
1972 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1973 {
1974         switch (fib_entry->fib_node->fib->proto) {
1975         case MLXSW_SP_L3_PROTO_IPV4:
1976                 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1977                 break;
1978         case MLXSW_SP_L3_PROTO_IPV6:
1979                 WARN_ON_ONCE(1);
1980         }
1981
1982         fib_entry->offloaded = false;
1983 }
1984
1985 static void
1986 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1987                                    enum mlxsw_reg_ralue_op op, int err)
1988 {
1989         switch (op) {
1990         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1991                 if (!fib_entry->offloaded)
1992                         return;
1993                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1994         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
1995                 if (err)
1996                         return;
1997                 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1998                     !fib_entry->offloaded)
1999                         mlxsw_sp_fib_entry_offload_set(fib_entry);
2000                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2001                          fib_entry->offloaded)
2002                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
2003                 return;
2004         default:
2005                 return;
2006         }
2007 }
2008
2009 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
2010                                          struct mlxsw_sp_fib_entry *fib_entry,
2011                                          enum mlxsw_reg_ralue_op op)
2012 {
2013         char ralue_pl[MLXSW_REG_RALUE_LEN];
2014         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2015         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2016         enum mlxsw_reg_ralue_trap_action trap_action;
2017         u16 trap_id = 0;
2018         u32 adjacency_index = 0;
2019         u16 ecmp_size = 0;
2020
2021         /* In case the nexthop group adjacency index is valid, use it
2022          * with provided ECMP size. Otherwise, setup trap and pass
2023          * traffic to kernel.
2024          */
2025         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2026                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2027                 adjacency_index = fib_entry->nh_group->adj_index;
2028                 ecmp_size = fib_entry->nh_group->ecmp_size;
2029         } else {
2030                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2031                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2032         }
2033
2034         mlxsw_reg_ralue_pack4(ralue_pl,
2035                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2036                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
2037                               *p_dip);
2038         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2039                                         adjacency_index, ecmp_size);
2040         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2041 }
2042
2043 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
2044                                         struct mlxsw_sp_fib_entry *fib_entry,
2045                                         enum mlxsw_reg_ralue_op op)
2046 {
2047         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
2048         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2049         enum mlxsw_reg_ralue_trap_action trap_action;
2050         char ralue_pl[MLXSW_REG_RALUE_LEN];
2051         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2052         u16 trap_id = 0;
2053         u16 rif_index = 0;
2054
2055         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2056                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2057                 rif_index = rif->rif_index;
2058         } else {
2059                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2060                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2061         }
2062
2063         mlxsw_reg_ralue_pack4(ralue_pl,
2064                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2065                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
2066                               *p_dip);
2067         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
2068                                        rif_index);
2069         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2070 }
2071
2072 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
2073                                        struct mlxsw_sp_fib_entry *fib_entry,
2074                                        enum mlxsw_reg_ralue_op op)
2075 {
2076         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2077         char ralue_pl[MLXSW_REG_RALUE_LEN];
2078         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2079
2080         mlxsw_reg_ralue_pack4(ralue_pl,
2081                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2082                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
2083                               *p_dip);
2084         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2085         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2086 }
2087
2088 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
2089                                   struct mlxsw_sp_fib_entry *fib_entry,
2090                                   enum mlxsw_reg_ralue_op op)
2091 {
2092         switch (fib_entry->type) {
2093         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2094                 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
2095         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2096                 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
2097         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
2098                 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
2099         }
2100         return -EINVAL;
2101 }
2102
2103 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2104                                  struct mlxsw_sp_fib_entry *fib_entry,
2105                                  enum mlxsw_reg_ralue_op op)
2106 {
2107         int err = -EINVAL;
2108
2109         switch (fib_entry->fib_node->fib->proto) {
2110         case MLXSW_SP_L3_PROTO_IPV4:
2111                 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
2112                 break;
2113         case MLXSW_SP_L3_PROTO_IPV6:
2114                 return err;
2115         }
2116         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
2117         return err;
2118 }
2119
2120 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2121                                      struct mlxsw_sp_fib_entry *fib_entry)
2122 {
2123         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2124                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
2125 }
2126
2127 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
2128                                   struct mlxsw_sp_fib_entry *fib_entry)
2129 {
2130         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2131                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
2132 }
2133
2134 static int
2135 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
2136                              const struct fib_entry_notifier_info *fen_info,
2137                              struct mlxsw_sp_fib_entry *fib_entry)
2138 {
2139         struct fib_info *fi = fen_info->fi;
2140
2141         switch (fen_info->type) {
2142         case RTN_BROADCAST: /* fall through */
2143         case RTN_LOCAL:
2144                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2145                 return 0;
2146         case RTN_UNREACHABLE: /* fall through */
2147         case RTN_BLACKHOLE: /* fall through */
2148         case RTN_PROHIBIT:
2149                 /* Packets hitting these routes need to be trapped, but
2150                  * can do so with a lower priority than packets directed
2151                  * at the host, so use action type local instead of trap.
2152                  */
2153                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2154                 return 0;
2155         case RTN_UNICAST:
2156                 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
2157                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2158                 else
2159                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
2160                 return 0;
2161         default:
2162                 return -EINVAL;
2163         }
2164 }
2165
2166 static struct mlxsw_sp_fib_entry *
2167 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
2168                            struct mlxsw_sp_fib_node *fib_node,
2169                            const struct fib_entry_notifier_info *fen_info)
2170 {
2171         struct mlxsw_sp_fib_entry *fib_entry;
2172         int err;
2173
2174         fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
2175         if (!fib_entry) {
2176                 err = -ENOMEM;
2177                 goto err_fib_entry_alloc;
2178         }
2179
2180         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
2181         if (err)
2182                 goto err_fib4_entry_type_set;
2183
2184         err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
2185         if (err)
2186                 goto err_nexthop_group_get;
2187
2188         fib_entry->params.prio = fen_info->fi->fib_priority;
2189         fib_entry->params.tb_id = fen_info->tb_id;
2190         fib_entry->params.type = fen_info->type;
2191         fib_entry->params.tos = fen_info->tos;
2192
2193         fib_entry->fib_node = fib_node;
2194
2195         return fib_entry;
2196
2197 err_nexthop_group_get:
2198 err_fib4_entry_type_set:
2199         kfree(fib_entry);
2200 err_fib_entry_alloc:
2201         return ERR_PTR(err);
2202 }
2203
2204 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2205                                         struct mlxsw_sp_fib_entry *fib_entry)
2206 {
2207         mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
2208         kfree(fib_entry);
2209 }
2210
2211 static struct mlxsw_sp_fib_node *
2212 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2213                        const struct fib_entry_notifier_info *fen_info);
2214
2215 static struct mlxsw_sp_fib_entry *
2216 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
2217                            const struct fib_entry_notifier_info *fen_info)
2218 {
2219         struct mlxsw_sp_fib_entry *fib_entry;
2220         struct mlxsw_sp_fib_node *fib_node;
2221
2222         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2223         if (IS_ERR(fib_node))
2224                 return NULL;
2225
2226         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2227                 if (fib_entry->params.tb_id == fen_info->tb_id &&
2228                     fib_entry->params.tos == fen_info->tos &&
2229                     fib_entry->params.type == fen_info->type &&
2230                     fib_entry->nh_group->key.fi == fen_info->fi) {
2231                         return fib_entry;
2232                 }
2233         }
2234
2235         return NULL;
2236 }
2237
2238 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
2239         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
2240         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
2241         .key_len = sizeof(struct mlxsw_sp_fib_key),
2242         .automatic_shrinking = true,
2243 };
2244
2245 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
2246                                     struct mlxsw_sp_fib_node *fib_node)
2247 {
2248         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
2249                                       mlxsw_sp_fib_ht_params);
2250 }
2251
2252 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
2253                                      struct mlxsw_sp_fib_node *fib_node)
2254 {
2255         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
2256                                mlxsw_sp_fib_ht_params);
2257 }
2258
2259 static struct mlxsw_sp_fib_node *
2260 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2261                          size_t addr_len, unsigned char prefix_len)
2262 {
2263         struct mlxsw_sp_fib_key key;
2264
2265         memset(&key, 0, sizeof(key));
2266         memcpy(key.addr, addr, addr_len);
2267         key.prefix_len = prefix_len;
2268         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2269 }
2270
2271 static struct mlxsw_sp_fib_node *
2272 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
2273                          size_t addr_len, unsigned char prefix_len)
2274 {
2275         struct mlxsw_sp_fib_node *fib_node;
2276
2277         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2278         if (!fib_node)
2279                 return NULL;
2280
2281         INIT_LIST_HEAD(&fib_node->entry_list);
2282         list_add(&fib_node->list, &fib->node_list);
2283         memcpy(fib_node->key.addr, addr, addr_len);
2284         fib_node->key.prefix_len = prefix_len;
2285
2286         return fib_node;
2287 }
2288
2289 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2290 {
2291         list_del(&fib_node->list);
2292         WARN_ON(!list_empty(&fib_node->entry_list));
2293         kfree(fib_node);
2294 }
2295
2296 static bool
2297 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2298                                  const struct mlxsw_sp_fib_entry *fib_entry)
2299 {
2300         return list_first_entry(&fib_node->entry_list,
2301                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
2302 }
2303
2304 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2305 {
2306         unsigned char prefix_len = fib_node->key.prefix_len;
2307         struct mlxsw_sp_fib *fib = fib_node->fib;
2308
2309         if (fib->prefix_ref_count[prefix_len]++ == 0)
2310                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2311 }
2312
2313 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2314 {
2315         unsigned char prefix_len = fib_node->key.prefix_len;
2316         struct mlxsw_sp_fib *fib = fib_node->fib;
2317
2318         if (--fib->prefix_ref_count[prefix_len] == 0)
2319                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2320 }
2321
2322 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2323                                   struct mlxsw_sp_fib_node *fib_node,
2324                                   struct mlxsw_sp_fib *fib)
2325 {
2326         struct mlxsw_sp_prefix_usage req_prefix_usage;
2327         struct mlxsw_sp_lpm_tree *lpm_tree;
2328         int err;
2329
2330         err = mlxsw_sp_fib_node_insert(fib, fib_node);
2331         if (err)
2332                 return err;
2333         fib_node->fib = fib;
2334
2335         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2336         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2337
2338         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2339                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2340                                                  &req_prefix_usage);
2341                 if (err)
2342                         goto err_tree_check;
2343         } else {
2344                 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2345                                                  fib->proto);
2346                 if (IS_ERR(lpm_tree))
2347                         return PTR_ERR(lpm_tree);
2348                 fib->lpm_tree = lpm_tree;
2349                 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2350                 if (err)
2351                         goto err_tree_bind;
2352         }
2353
2354         mlxsw_sp_fib_node_prefix_inc(fib_node);
2355
2356         return 0;
2357
2358 err_tree_bind:
2359         fib->lpm_tree = NULL;
2360         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2361 err_tree_check:
2362         fib_node->fib = NULL;
2363         mlxsw_sp_fib_node_remove(fib, fib_node);
2364         return err;
2365 }
2366
2367 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2368                                    struct mlxsw_sp_fib_node *fib_node)
2369 {
2370         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2371         struct mlxsw_sp_fib *fib = fib_node->fib;
2372
2373         mlxsw_sp_fib_node_prefix_dec(fib_node);
2374
2375         if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2376                 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2377                 fib->lpm_tree = NULL;
2378                 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2379         } else {
2380                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2381         }
2382
2383         fib_node->fib = NULL;
2384         mlxsw_sp_fib_node_remove(fib, fib_node);
2385 }
2386
2387 static struct mlxsw_sp_fib_node *
2388 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2389                        const struct fib_entry_notifier_info *fen_info)
2390 {
2391         struct mlxsw_sp_fib_node *fib_node;
2392         struct mlxsw_sp_fib *fib;
2393         struct mlxsw_sp_vr *vr;
2394         int err;
2395
2396         vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2397         if (IS_ERR(vr))
2398                 return ERR_CAST(vr);
2399         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2400
2401         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2402                                             sizeof(fen_info->dst),
2403                                             fen_info->dst_len);
2404         if (fib_node)
2405                 return fib_node;
2406
2407         fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2408                                             sizeof(fen_info->dst),
2409                                             fen_info->dst_len);
2410         if (!fib_node) {
2411                 err = -ENOMEM;
2412                 goto err_fib_node_create;
2413         }
2414
2415         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2416         if (err)
2417                 goto err_fib_node_init;
2418
2419         return fib_node;
2420
2421 err_fib_node_init:
2422         mlxsw_sp_fib_node_destroy(fib_node);
2423 err_fib_node_create:
2424         mlxsw_sp_vr_put(vr);
2425         return ERR_PTR(err);
2426 }
2427
2428 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2429                                    struct mlxsw_sp_fib_node *fib_node)
2430 {
2431         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2432
2433         if (!list_empty(&fib_node->entry_list))
2434                 return;
2435         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2436         mlxsw_sp_fib_node_destroy(fib_node);
2437         mlxsw_sp_vr_put(vr);
2438 }
2439
2440 static struct mlxsw_sp_fib_entry *
2441 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2442                               const struct mlxsw_sp_fib_entry_params *params)
2443 {
2444         struct mlxsw_sp_fib_entry *fib_entry;
2445
2446         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2447                 if (fib_entry->params.tb_id > params->tb_id)
2448                         continue;
2449                 if (fib_entry->params.tb_id != params->tb_id)
2450                         break;
2451                 if (fib_entry->params.tos > params->tos)
2452                         continue;
2453                 if (fib_entry->params.prio >= params->prio ||
2454                     fib_entry->params.tos < params->tos)
2455                         return fib_entry;
2456         }
2457
2458         return NULL;
2459 }
2460
2461 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2462                                           struct mlxsw_sp_fib_entry *new_entry)
2463 {
2464         struct mlxsw_sp_fib_node *fib_node;
2465
2466         if (WARN_ON(!fib_entry))
2467                 return -EINVAL;
2468
2469         fib_node = fib_entry->fib_node;
2470         list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2471                 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2472                     fib_entry->params.tos != new_entry->params.tos ||
2473                     fib_entry->params.prio != new_entry->params.prio)
2474                         break;
2475         }
2476
2477         list_add_tail(&new_entry->list, &fib_entry->list);
2478         return 0;
2479 }
2480
2481 static int
2482 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2483                                struct mlxsw_sp_fib_entry *new_entry,
2484                                bool replace, bool append)
2485 {
2486         struct mlxsw_sp_fib_entry *fib_entry;
2487
2488         fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2489
2490         if (append)
2491                 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2492         if (replace && WARN_ON(!fib_entry))
2493                 return -EINVAL;
2494
2495         /* Insert new entry before replaced one, so that we can later
2496          * remove the second.
2497          */
2498         if (fib_entry) {
2499                 list_add_tail(&new_entry->list, &fib_entry->list);
2500         } else {
2501                 struct mlxsw_sp_fib_entry *last;
2502
2503                 list_for_each_entry(last, &fib_node->entry_list, list) {
2504                         if (new_entry->params.tb_id > last->params.tb_id)
2505                                 break;
2506                         fib_entry = last;
2507                 }
2508
2509                 if (fib_entry)
2510                         list_add(&new_entry->list, &fib_entry->list);
2511                 else
2512                         list_add(&new_entry->list, &fib_node->entry_list);
2513         }
2514
2515         return 0;
2516 }
2517
2518 static void
2519 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2520 {
2521         list_del(&fib_entry->list);
2522 }
2523
2524 static int
2525 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2526                              const struct mlxsw_sp_fib_node *fib_node,
2527                              struct mlxsw_sp_fib_entry *fib_entry)
2528 {
2529         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2530                 return 0;
2531
2532         /* To prevent packet loss, overwrite the previously offloaded
2533          * entry.
2534          */
2535         if (!list_is_singular(&fib_node->entry_list)) {
2536                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2537                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2538
2539                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2540         }
2541
2542         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2543 }
2544
2545 static void
2546 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2547                              const struct mlxsw_sp_fib_node *fib_node,
2548                              struct mlxsw_sp_fib_entry *fib_entry)
2549 {
2550         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2551                 return;
2552
2553         /* Promote the next entry by overwriting the deleted entry */
2554         if (!list_is_singular(&fib_node->entry_list)) {
2555                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2556                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2557
2558                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2559                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2560                 return;
2561         }
2562
2563         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2564 }
2565
2566 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2567                                          struct mlxsw_sp_fib_entry *fib_entry,
2568                                          bool replace, bool append)
2569 {
2570         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2571         int err;
2572
2573         err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2574                                              append);
2575         if (err)
2576                 return err;
2577
2578         err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2579         if (err)
2580                 goto err_fib4_node_entry_add;
2581
2582         return 0;
2583
2584 err_fib4_node_entry_add:
2585         mlxsw_sp_fib4_node_list_remove(fib_entry);
2586         return err;
2587 }
2588
2589 static void
2590 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2591                                 struct mlxsw_sp_fib_entry *fib_entry)
2592 {
2593         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2594
2595         mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2596         mlxsw_sp_fib4_node_list_remove(fib_entry);
2597 }
2598
2599 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2600                                         struct mlxsw_sp_fib_entry *fib_entry,
2601                                         bool replace)
2602 {
2603         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2604         struct mlxsw_sp_fib_entry *replaced;
2605
2606         if (!replace)
2607                 return;
2608
2609         /* We inserted the new entry before replaced one */
2610         replaced = list_next_entry(fib_entry, list);
2611
2612         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2613         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2614         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2615 }
2616
2617 static int
2618 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2619                          const struct fib_entry_notifier_info *fen_info,
2620                          bool replace, bool append)
2621 {
2622         struct mlxsw_sp_fib_entry *fib_entry;
2623         struct mlxsw_sp_fib_node *fib_node;
2624         int err;
2625
2626         if (mlxsw_sp->router->aborted)
2627                 return 0;
2628
2629         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2630         if (IS_ERR(fib_node)) {
2631                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2632                 return PTR_ERR(fib_node);
2633         }
2634
2635         fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2636         if (IS_ERR(fib_entry)) {
2637                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2638                 err = PTR_ERR(fib_entry);
2639                 goto err_fib4_entry_create;
2640         }
2641
2642         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2643                                             append);
2644         if (err) {
2645                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2646                 goto err_fib4_node_entry_link;
2647         }
2648
2649         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2650
2651         return 0;
2652
2653 err_fib4_node_entry_link:
2654         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2655 err_fib4_entry_create:
2656         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2657         return err;
2658 }
2659
2660 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2661                                      struct fib_entry_notifier_info *fen_info)
2662 {
2663         struct mlxsw_sp_fib_entry *fib_entry;
2664         struct mlxsw_sp_fib_node *fib_node;
2665
2666         if (mlxsw_sp->router->aborted)
2667                 return;
2668
2669         fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2670         if (WARN_ON(!fib_entry))
2671                 return;
2672         fib_node = fib_entry->fib_node;
2673
2674         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2675         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2676         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2677 }
2678
2679 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2680 {
2681         char ralta_pl[MLXSW_REG_RALTA_LEN];
2682         char ralst_pl[MLXSW_REG_RALST_LEN];
2683         int i, err;
2684
2685         mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2686                              MLXSW_SP_LPM_TREE_MIN);
2687         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2688         if (err)
2689                 return err;
2690
2691         mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2692         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2693         if (err)
2694                 return err;
2695
2696         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2697                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
2698                 char raltb_pl[MLXSW_REG_RALTB_LEN];
2699                 char ralue_pl[MLXSW_REG_RALUE_LEN];
2700
2701                 if (!mlxsw_sp_vr_is_used(vr))
2702                         continue;
2703
2704                 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2705                                      MLXSW_REG_RALXX_PROTOCOL_IPV4,
2706                                      MLXSW_SP_LPM_TREE_MIN);
2707                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2708                                       raltb_pl);
2709                 if (err)
2710                         return err;
2711
2712                 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2713                                       MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2714                                       0);
2715                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2716                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2717                                       ralue_pl);
2718                 if (err)
2719                         return err;
2720         }
2721
2722         return 0;
2723 }
2724
2725 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2726                                      struct mlxsw_sp_fib_node *fib_node)
2727 {
2728         struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2729
2730         list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2731                 bool do_break = &tmp->list == &fib_node->entry_list;
2732
2733                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2734                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2735                 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2736                 /* Break when entry list is empty and node was freed.
2737                  * Otherwise, we'll access freed memory in the next
2738                  * iteration.
2739                  */
2740                 if (do_break)
2741                         break;
2742         }
2743 }
2744
2745 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2746                                     struct mlxsw_sp_fib_node *fib_node)
2747 {
2748         switch (fib_node->fib->proto) {
2749         case MLXSW_SP_L3_PROTO_IPV4:
2750                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2751                 break;
2752         case MLXSW_SP_L3_PROTO_IPV6:
2753                 WARN_ON_ONCE(1);
2754                 break;
2755         }
2756 }
2757
2758 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2759                                   struct mlxsw_sp_vr *vr,
2760                                   enum mlxsw_sp_l3proto proto)
2761 {
2762         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2763         struct mlxsw_sp_fib_node *fib_node, *tmp;
2764
2765         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2766                 bool do_break = &tmp->list == &fib->node_list;
2767
2768                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2769                 if (do_break)
2770                         break;
2771         }
2772 }
2773
2774 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2775 {
2776         int i;
2777
2778         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2779                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
2780
2781                 if (!mlxsw_sp_vr_is_used(vr))
2782                         continue;
2783                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2784         }
2785 }
2786
2787 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2788 {
2789         int err;
2790
2791         if (mlxsw_sp->router->aborted)
2792                 return;
2793         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2794         mlxsw_sp_router_fib_flush(mlxsw_sp);
2795         mlxsw_sp->router->aborted = true;
2796         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2797         if (err)
2798                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2799 }
2800
2801 struct mlxsw_sp_fib_event_work {
2802         struct work_struct work;
2803         union {
2804                 struct fib_entry_notifier_info fen_info;
2805                 struct fib_rule_notifier_info fr_info;
2806                 struct fib_nh_notifier_info fnh_info;
2807         };
2808         struct mlxsw_sp *mlxsw_sp;
2809         unsigned long event;
2810 };
2811
2812 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2813 {
2814         struct mlxsw_sp_fib_event_work *fib_work =
2815                 container_of(work, struct mlxsw_sp_fib_event_work, work);
2816         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2817         struct fib_rule *rule;
2818         bool replace, append;
2819         int err;
2820
2821         /* Protect internal structures from changes */
2822         rtnl_lock();
2823         switch (fib_work->event) {
2824         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2825         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2826         case FIB_EVENT_ENTRY_ADD:
2827                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2828                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2829                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2830                                                replace, append);
2831                 if (err)
2832                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2833                 fib_info_put(fib_work->fen_info.fi);
2834                 break;
2835         case FIB_EVENT_ENTRY_DEL:
2836                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2837                 fib_info_put(fib_work->fen_info.fi);
2838                 break;
2839         case FIB_EVENT_RULE_ADD: /* fall through */
2840         case FIB_EVENT_RULE_DEL:
2841                 rule = fib_work->fr_info.rule;
2842                 if (!fib4_rule_default(rule) && !rule->l3mdev)
2843                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2844                 fib_rule_put(rule);
2845                 break;
2846         case FIB_EVENT_NH_ADD: /* fall through */
2847         case FIB_EVENT_NH_DEL:
2848                 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2849                                        fib_work->fnh_info.fib_nh);
2850                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2851                 break;
2852         }
2853         rtnl_unlock();
2854         kfree(fib_work);
2855 }
2856
2857 /* Called with rcu_read_lock() */
2858 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2859                                      unsigned long event, void *ptr)
2860 {
2861         struct mlxsw_sp_fib_event_work *fib_work;
2862         struct fib_notifier_info *info = ptr;
2863         struct mlxsw_sp_router *router;
2864
2865         if (!net_eq(info->net, &init_net))
2866                 return NOTIFY_DONE;
2867
2868         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2869         if (WARN_ON(!fib_work))
2870                 return NOTIFY_BAD;
2871
2872         INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2873         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
2874         fib_work->mlxsw_sp = router->mlxsw_sp;
2875         fib_work->event = event;
2876
2877         switch (event) {
2878         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2879         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2880         case FIB_EVENT_ENTRY_ADD: /* fall through */
2881         case FIB_EVENT_ENTRY_DEL:
2882                 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2883                 /* Take referece on fib_info to prevent it from being
2884                  * freed while work is queued. Release it afterwards.
2885                  */
2886                 fib_info_hold(fib_work->fen_info.fi);
2887                 break;
2888         case FIB_EVENT_RULE_ADD: /* fall through */
2889         case FIB_EVENT_RULE_DEL:
2890                 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2891                 fib_rule_get(fib_work->fr_info.rule);
2892                 break;
2893         case FIB_EVENT_NH_ADD: /* fall through */
2894         case FIB_EVENT_NH_DEL:
2895                 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2896                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2897                 break;
2898         }
2899
2900         mlxsw_core_schedule_work(&fib_work->work);
2901
2902         return NOTIFY_DONE;
2903 }
2904
2905 static struct mlxsw_sp_rif *
2906 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2907                          const struct net_device *dev)
2908 {
2909         int i;
2910
2911         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2912                 if (mlxsw_sp->router->rifs[i] &&
2913                     mlxsw_sp->router->rifs[i]->dev == dev)
2914                         return mlxsw_sp->router->rifs[i];
2915
2916         return NULL;
2917 }
2918
2919 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2920 {
2921         char ritr_pl[MLXSW_REG_RITR_LEN];
2922         int err;
2923
2924         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2925         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2926         if (WARN_ON_ONCE(err))
2927                 return err;
2928
2929         mlxsw_reg_ritr_enable_set(ritr_pl, false);
2930         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2931 }
2932
2933 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2934                                           struct mlxsw_sp_rif *rif)
2935 {
2936         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
2937         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
2938         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
2939 }
2940
2941 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif,
2942                                        const struct in_device *in_dev,
2943                                        unsigned long event)
2944 {
2945         switch (event) {
2946         case NETDEV_UP:
2947                 if (!rif)
2948                         return true;
2949                 return false;
2950         case NETDEV_DOWN:
2951                 if (rif && !in_dev->ifa_list &&
2952                     !netif_is_l3_slave(rif->dev))
2953                         return true;
2954                 /* It is possible we already removed the RIF ourselves
2955                  * if it was assigned to a netdev that is now a bridge
2956                  * or LAG slave.
2957                  */
2958                 return false;
2959         }
2960
2961         return false;
2962 }
2963
2964 static enum mlxsw_sp_rif_type
2965 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
2966                       const struct net_device *dev)
2967 {
2968         enum mlxsw_sp_fid_type type;
2969
2970         /* RIF type is derived from the type of the underlying FID */
2971         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
2972                 type = MLXSW_SP_FID_TYPE_8021Q;
2973         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
2974                 type = MLXSW_SP_FID_TYPE_8021Q;
2975         else if (netif_is_bridge_master(dev))
2976                 type = MLXSW_SP_FID_TYPE_8021D;
2977         else
2978                 type = MLXSW_SP_FID_TYPE_RFID;
2979
2980         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
2981 }
2982
2983 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
2984 {
2985         int i;
2986
2987         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
2988                 if (!mlxsw_sp->router->rifs[i]) {
2989                         *p_rif_index = i;
2990                         return 0;
2991                 }
2992         }
2993
2994         return -ENOBUFS;
2995 }
2996
2997 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
2998                                                u16 vr_id,
2999                                                struct net_device *l3_dev)
3000 {
3001         struct mlxsw_sp_rif *rif;
3002
3003         rif = kzalloc(rif_size, GFP_KERNEL);
3004         if (!rif)
3005                 return NULL;
3006
3007         INIT_LIST_HEAD(&rif->nexthop_list);
3008         INIT_LIST_HEAD(&rif->neigh_list);
3009         ether_addr_copy(rif->addr, l3_dev->dev_addr);
3010         rif->mtu = l3_dev->mtu;
3011         rif->vr_id = vr_id;
3012         rif->dev = l3_dev;
3013         rif->rif_index = rif_index;
3014
3015         return rif;
3016 }
3017
3018 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
3019                                            u16 rif_index)
3020 {
3021         return mlxsw_sp->router->rifs[rif_index];
3022 }
3023
3024 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
3025 {
3026         return rif->rif_index;
3027 }
3028
3029 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
3030 {
3031         return rif->dev->ifindex;
3032 }
3033
3034 static struct mlxsw_sp_rif *
3035 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
3036                     const struct mlxsw_sp_rif_params *params)
3037 {
3038         u32 tb_id = l3mdev_fib_table(params->dev);
3039         const struct mlxsw_sp_rif_ops *ops;
3040         enum mlxsw_sp_rif_type type;
3041         struct mlxsw_sp_rif *rif;
3042         struct mlxsw_sp_fid *fid;
3043         struct mlxsw_sp_vr *vr;
3044         u16 rif_index;
3045         int err;
3046
3047         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
3048         ops = mlxsw_sp->router->rif_ops_arr[type];
3049
3050         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3051         if (IS_ERR(vr))
3052                 return ERR_CAST(vr);
3053
3054         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
3055         if (err)
3056                 goto err_rif_index_alloc;
3057
3058         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
3059         if (!rif) {
3060                 err = -ENOMEM;
3061                 goto err_rif_alloc;
3062         }
3063         rif->mlxsw_sp = mlxsw_sp;
3064         rif->ops = ops;
3065
3066         fid = ops->fid_get(rif);
3067         if (IS_ERR(fid)) {
3068                 err = PTR_ERR(fid);
3069                 goto err_fid_get;
3070         }
3071         rif->fid = fid;
3072
3073         if (ops->setup)
3074                 ops->setup(rif, params);
3075
3076         err = ops->configure(rif);
3077         if (err)
3078                 goto err_configure;
3079
3080         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, params->dev->dev_addr,
3081                                   mlxsw_sp_fid_index(fid), true);
3082         if (err)
3083                 goto err_rif_fdb_op;
3084
3085         mlxsw_sp_rif_counters_alloc(rif);
3086         mlxsw_sp_fid_rif_set(fid, rif);
3087         mlxsw_sp->router->rifs[rif_index] = rif;
3088         vr->rif_count++;
3089
3090         return rif;
3091
3092 err_rif_fdb_op:
3093         ops->deconfigure(rif);
3094 err_configure:
3095         mlxsw_sp_fid_put(fid);
3096 err_fid_get:
3097         kfree(rif);
3098 err_rif_alloc:
3099 err_rif_index_alloc:
3100         mlxsw_sp_vr_put(vr);
3101         return ERR_PTR(err);
3102 }
3103
3104 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
3105 {
3106         const struct mlxsw_sp_rif_ops *ops = rif->ops;
3107         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3108         struct mlxsw_sp_fid *fid = rif->fid;
3109         struct mlxsw_sp_vr *vr;
3110
3111         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
3112         vr = &mlxsw_sp->router->vrs[rif->vr_id];
3113
3114         vr->rif_count--;
3115         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
3116         mlxsw_sp_fid_rif_set(fid, NULL);
3117         mlxsw_sp_rif_counters_free(rif);
3118         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->dev->dev_addr,
3119                             mlxsw_sp_fid_index(fid), false);
3120         ops->deconfigure(rif);
3121         mlxsw_sp_fid_put(fid);
3122         kfree(rif);
3123         mlxsw_sp_vr_put(vr);
3124 }
3125
3126 static void
3127 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
3128                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3129 {
3130         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3131
3132         params->vid = mlxsw_sp_port_vlan->vid;
3133         params->lag = mlxsw_sp_port->lagged;
3134         if (params->lag)
3135                 params->lag_id = mlxsw_sp_port->lag_id;
3136         else
3137                 params->system_port = mlxsw_sp_port->local_port;
3138 }
3139
3140 static int
3141 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
3142                                struct net_device *l3_dev)
3143 {
3144         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3145         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
3146         u16 vid = mlxsw_sp_port_vlan->vid;
3147         struct mlxsw_sp_rif *rif;
3148         struct mlxsw_sp_fid *fid;
3149         int err;
3150
3151         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3152         if (!rif) {
3153                 struct mlxsw_sp_rif_params params = {
3154                         .dev = l3_dev,
3155                 };
3156
3157                 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
3158                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
3159                 if (IS_ERR(rif))
3160                         return PTR_ERR(rif);
3161         }
3162
3163         /* FID was already created, just take a reference */
3164         fid = rif->ops->fid_get(rif);
3165         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
3166         if (err)
3167                 goto err_fid_port_vid_map;
3168
3169         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
3170         if (err)
3171                 goto err_port_vid_learning_set;
3172
3173         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
3174                                         BR_STATE_FORWARDING);
3175         if (err)
3176                 goto err_port_vid_stp_set;
3177
3178         mlxsw_sp_port_vlan->fid = fid;
3179
3180         return 0;
3181
3182 err_port_vid_stp_set:
3183         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
3184 err_port_vid_learning_set:
3185         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3186 err_fid_port_vid_map:
3187         mlxsw_sp_fid_put(fid);
3188         return err;
3189 }
3190
3191 void
3192 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3193 {
3194         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3195         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
3196         u16 vid = mlxsw_sp_port_vlan->vid;
3197
3198         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
3199                 return;
3200
3201         mlxsw_sp_port_vlan->fid = NULL;
3202         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
3203         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
3204         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3205         /* If router port holds the last reference on the rFID, then the
3206          * associated Sub-port RIF will be destroyed.
3207          */
3208         mlxsw_sp_fid_put(fid);
3209 }
3210
3211 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
3212                                              struct net_device *port_dev,
3213                                              unsigned long event, u16 vid)
3214 {
3215         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
3216         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
3217
3218         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
3219         if (WARN_ON(!mlxsw_sp_port_vlan))
3220                 return -EINVAL;
3221
3222         switch (event) {
3223         case NETDEV_UP:
3224                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
3225                                                       l3_dev);
3226         case NETDEV_DOWN:
3227                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
3228                 break;
3229         }
3230
3231         return 0;
3232 }
3233
3234 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
3235                                         unsigned long event)
3236 {
3237         if (netif_is_bridge_port(port_dev) ||
3238             netif_is_lag_port(port_dev) ||
3239             netif_is_ovs_port(port_dev))
3240                 return 0;
3241
3242         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
3243 }
3244
3245 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
3246                                          struct net_device *lag_dev,
3247                                          unsigned long event, u16 vid)
3248 {
3249         struct net_device *port_dev;
3250         struct list_head *iter;
3251         int err;
3252
3253         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
3254                 if (mlxsw_sp_port_dev_check(port_dev)) {
3255                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
3256                                                                 port_dev,
3257                                                                 event, vid);
3258                         if (err)
3259                                 return err;
3260                 }
3261         }
3262
3263         return 0;
3264 }
3265
3266 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
3267                                        unsigned long event)
3268 {
3269         if (netif_is_bridge_port(lag_dev))
3270                 return 0;
3271
3272         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
3273 }
3274
3275 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3276                                           unsigned long event)
3277 {
3278         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3279         struct mlxsw_sp_rif_params params = {
3280                 .dev = l3_dev,
3281         };
3282         struct mlxsw_sp_rif *rif;
3283
3284         switch (event) {
3285         case NETDEV_UP:
3286                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
3287                 if (IS_ERR(rif))
3288                         return PTR_ERR(rif);
3289                 break;
3290         case NETDEV_DOWN:
3291                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3292                 mlxsw_sp_rif_destroy(rif);
3293                 break;
3294         }
3295
3296         return 0;
3297 }
3298
3299 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3300                                         unsigned long event)
3301 {
3302         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3303         u16 vid = vlan_dev_vlan_id(vlan_dev);
3304
3305         if (netif_is_bridge_port(vlan_dev))
3306                 return 0;
3307
3308         if (mlxsw_sp_port_dev_check(real_dev))
3309                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
3310                                                          event, vid);
3311         else if (netif_is_lag_master(real_dev))
3312                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3313                                                      vid);
3314         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
3315                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
3316
3317         return 0;
3318 }
3319
3320 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
3321                                      unsigned long event)
3322 {
3323         if (mlxsw_sp_port_dev_check(dev))
3324                 return mlxsw_sp_inetaddr_port_event(dev, event);
3325         else if (netif_is_lag_master(dev))
3326                 return mlxsw_sp_inetaddr_lag_event(dev, event);
3327         else if (netif_is_bridge_master(dev))
3328                 return mlxsw_sp_inetaddr_bridge_event(dev, event);
3329         else if (is_vlan_dev(dev))
3330                 return mlxsw_sp_inetaddr_vlan_event(dev, event);
3331         else
3332                 return 0;
3333 }
3334
3335 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3336                             unsigned long event, void *ptr)
3337 {
3338         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3339         struct net_device *dev = ifa->ifa_dev->dev;
3340         struct mlxsw_sp *mlxsw_sp;
3341         struct mlxsw_sp_rif *rif;
3342         int err = 0;
3343
3344         mlxsw_sp = mlxsw_sp_lower_get(dev);
3345         if (!mlxsw_sp)
3346                 goto out;
3347
3348         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3349         if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event))
3350                 goto out;
3351
3352         err = __mlxsw_sp_inetaddr_event(dev, event);
3353 out:
3354         return notifier_from_errno(err);
3355 }
3356
3357 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
3358                              const char *mac, int mtu)
3359 {
3360         char ritr_pl[MLXSW_REG_RITR_LEN];
3361         int err;
3362
3363         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
3364         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3365         if (err)
3366                 return err;
3367
3368         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3369         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3370         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3371         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3372 }
3373
3374 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3375 {
3376         struct mlxsw_sp *mlxsw_sp;
3377         struct mlxsw_sp_rif *rif;
3378         u16 fid_index;
3379         int err;
3380
3381         mlxsw_sp = mlxsw_sp_lower_get(dev);
3382         if (!mlxsw_sp)
3383                 return 0;
3384
3385         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3386         if (!rif)
3387                 return 0;
3388         fid_index = mlxsw_sp_fid_index(rif->fid);
3389
3390         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
3391         if (err)
3392                 return err;
3393
3394         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
3395                                 dev->mtu);
3396         if (err)
3397                 goto err_rif_edit;
3398
3399         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
3400         if (err)
3401                 goto err_rif_fdb_op;
3402
3403         ether_addr_copy(rif->addr, dev->dev_addr);
3404         rif->mtu = dev->mtu;
3405
3406         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
3407
3408         return 0;
3409
3410 err_rif_fdb_op:
3411         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
3412 err_rif_edit:
3413         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
3414         return err;
3415 }
3416
3417 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
3418                                   struct net_device *l3_dev)
3419 {
3420         struct mlxsw_sp_rif *rif;
3421
3422         /* If netdev is already associated with a RIF, then we need to
3423          * destroy it and create a new one with the new virtual router ID.
3424          */
3425         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3426         if (rif)
3427                 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
3428
3429         return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
3430 }
3431
3432 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
3433                                     struct net_device *l3_dev)
3434 {
3435         struct mlxsw_sp_rif *rif;
3436
3437         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3438         if (!rif)
3439                 return;
3440         __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
3441 }
3442
3443 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
3444                                  struct netdev_notifier_changeupper_info *info)
3445 {
3446         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3447         int err = 0;
3448
3449         if (!mlxsw_sp)
3450                 return 0;
3451
3452         switch (event) {
3453         case NETDEV_PRECHANGEUPPER:
3454                 return 0;
3455         case NETDEV_CHANGEUPPER:
3456                 if (info->linking)
3457                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
3458                 else
3459                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
3460                 break;
3461         }
3462
3463         return err;
3464 }
3465
3466 static struct mlxsw_sp_rif_subport *
3467 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
3468 {
3469         return container_of(rif, struct mlxsw_sp_rif_subport, common);
3470 }
3471
3472 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
3473                                        const struct mlxsw_sp_rif_params *params)
3474 {
3475         struct mlxsw_sp_rif_subport *rif_subport;
3476
3477         rif_subport = mlxsw_sp_rif_subport_rif(rif);
3478         rif_subport->vid = params->vid;
3479         rif_subport->lag = params->lag;
3480         if (params->lag)
3481                 rif_subport->lag_id = params->lag_id;
3482         else
3483                 rif_subport->system_port = params->system_port;
3484 }
3485
3486 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
3487 {
3488         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3489         struct mlxsw_sp_rif_subport *rif_subport;
3490         char ritr_pl[MLXSW_REG_RITR_LEN];
3491
3492         rif_subport = mlxsw_sp_rif_subport_rif(rif);
3493         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
3494                             rif->rif_index, rif->vr_id, rif->dev->mtu,
3495                             rif->dev->dev_addr);
3496         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
3497                                   rif_subport->lag ? rif_subport->lag_id :
3498                                                      rif_subport->system_port,
3499                                   rif_subport->vid);
3500
3501         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3502 }
3503
3504 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
3505 {
3506         return mlxsw_sp_rif_subport_op(rif, true);
3507 }
3508
3509 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
3510 {
3511         mlxsw_sp_rif_subport_op(rif, false);
3512 }
3513
3514 static struct mlxsw_sp_fid *
3515 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
3516 {
3517         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
3518 }
3519
3520 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
3521         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
3522         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
3523         .setup                  = mlxsw_sp_rif_subport_setup,
3524         .configure              = mlxsw_sp_rif_subport_configure,
3525         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
3526         .fid_get                = mlxsw_sp_rif_subport_fid_get,
3527 };
3528
3529 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
3530                                     enum mlxsw_reg_ritr_if_type type,
3531                                     u16 vid_fid, bool enable)
3532 {
3533         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3534         char ritr_pl[MLXSW_REG_RITR_LEN];
3535
3536         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
3537                             rif->dev->mtu, rif->dev->dev_addr);
3538         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
3539
3540         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3541 }
3542
3543 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
3544 {
3545         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
3546 }
3547
3548 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
3549 {
3550         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3551         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3552         int err;
3553
3554         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
3555         if (err)
3556                 return err;
3557
3558         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3559                                      mlxsw_sp_router_port(mlxsw_sp), true);
3560         if (err)
3561                 goto err_fid_bc_flood_set;
3562
3563         return 0;
3564
3565 err_fid_bc_flood_set:
3566         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3567         return err;
3568 }
3569
3570 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
3571 {
3572         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3573         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3574
3575         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3576                                mlxsw_sp_router_port(mlxsw_sp), false);
3577         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3578 }
3579
3580 static struct mlxsw_sp_fid *
3581 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
3582 {
3583         u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
3584
3585         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
3586 }
3587
3588 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
3589         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
3590         .rif_size               = sizeof(struct mlxsw_sp_rif),
3591         .configure              = mlxsw_sp_rif_vlan_configure,
3592         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
3593         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
3594 };
3595
3596 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
3597 {
3598         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3599         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3600         int err;
3601
3602         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
3603                                        true);
3604         if (err)
3605                 return err;
3606
3607         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3608                                      mlxsw_sp_router_port(mlxsw_sp), true);
3609         if (err)
3610                 goto err_fid_bc_flood_set;
3611
3612         return 0;
3613
3614 err_fid_bc_flood_set:
3615         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3616         return err;
3617 }
3618
3619 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
3620 {
3621         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3622         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3623
3624         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3625                                mlxsw_sp_router_port(mlxsw_sp), false);
3626         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3627 }
3628
3629 static struct mlxsw_sp_fid *
3630 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
3631 {
3632         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
3633 }
3634
3635 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
3636         .type                   = MLXSW_SP_RIF_TYPE_FID,
3637         .rif_size               = sizeof(struct mlxsw_sp_rif),
3638         .configure              = mlxsw_sp_rif_fid_configure,
3639         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
3640         .fid_get                = mlxsw_sp_rif_fid_fid_get,
3641 };
3642
3643 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
3644         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
3645         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_ops,
3646         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
3647 };
3648
3649 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
3650 {
3651         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3652
3653         mlxsw_sp->router->rifs = kcalloc(max_rifs,
3654                                          sizeof(struct mlxsw_sp_rif *),
3655                                          GFP_KERNEL);
3656         if (!mlxsw_sp->router->rifs)
3657                 return -ENOMEM;
3658
3659         mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
3660
3661         return 0;
3662 }
3663
3664 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
3665 {
3666         int i;
3667
3668         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3669                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
3670
3671         kfree(mlxsw_sp->router->rifs);
3672 }
3673
3674 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3675 {
3676         struct mlxsw_sp_router *router;
3677
3678         /* Flush pending FIB notifications and then flush the device's
3679          * table before requesting another dump. The FIB notification
3680          * block is unregistered, so no need to take RTNL.
3681          */
3682         mlxsw_core_flush_owq();
3683         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
3684         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
3685 }
3686
3687 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3688 {
3689         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3690         u64 max_rifs;
3691         int err;
3692
3693         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3694                 return -EIO;
3695         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3696
3697         mlxsw_reg_rgcr_pack(rgcr_pl, true);
3698         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3699         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3700         if (err)
3701                 return err;
3702         return 0;
3703 }
3704
3705 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3706 {
3707         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3708
3709         mlxsw_reg_rgcr_pack(rgcr_pl, false);
3710         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3711 }
3712
3713 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3714 {
3715         struct mlxsw_sp_router *router;
3716         int err;
3717
3718         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
3719         if (!router)
3720                 return -ENOMEM;
3721         mlxsw_sp->router = router;
3722         router->mlxsw_sp = mlxsw_sp;
3723
3724         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
3725         err = __mlxsw_sp_router_init(mlxsw_sp);
3726         if (err)
3727                 goto err_router_init;
3728
3729         err = mlxsw_sp_rifs_init(mlxsw_sp);
3730         if (err)
3731                 goto err_rifs_init;
3732
3733         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
3734                               &mlxsw_sp_nexthop_ht_params);
3735         if (err)
3736                 goto err_nexthop_ht_init;
3737
3738         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
3739                               &mlxsw_sp_nexthop_group_ht_params);
3740         if (err)
3741                 goto err_nexthop_group_ht_init;
3742
3743         err = mlxsw_sp_lpm_init(mlxsw_sp);
3744         if (err)
3745                 goto err_lpm_init;
3746
3747         err = mlxsw_sp_vrs_init(mlxsw_sp);
3748         if (err)
3749                 goto err_vrs_init;
3750
3751         err = mlxsw_sp_neigh_init(mlxsw_sp);
3752         if (err)
3753                 goto err_neigh_init;
3754
3755         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3756         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
3757                                     mlxsw_sp_router_fib_dump_flush);
3758         if (err)
3759                 goto err_register_fib_notifier;
3760
3761         return 0;
3762
3763 err_register_fib_notifier:
3764         mlxsw_sp_neigh_fini(mlxsw_sp);
3765 err_neigh_init:
3766         mlxsw_sp_vrs_fini(mlxsw_sp);
3767 err_vrs_init:
3768         mlxsw_sp_lpm_fini(mlxsw_sp);
3769 err_lpm_init:
3770         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3771 err_nexthop_group_ht_init:
3772         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
3773 err_nexthop_ht_init:
3774         mlxsw_sp_rifs_fini(mlxsw_sp);
3775 err_rifs_init:
3776         __mlxsw_sp_router_fini(mlxsw_sp);
3777 err_router_init:
3778         kfree(mlxsw_sp->router);
3779         return err;
3780 }
3781
3782 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3783 {
3784         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
3785         mlxsw_sp_neigh_fini(mlxsw_sp);
3786         mlxsw_sp_vrs_fini(mlxsw_sp);
3787         mlxsw_sp_lpm_fini(mlxsw_sp);
3788         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3789         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
3790         mlxsw_sp_rifs_fini(mlxsw_sp);
3791         __mlxsw_sp_router_fini(mlxsw_sp);
3792         kfree(mlxsw_sp->router);
3793 }