drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <linux/inetdevice.h>
  44 #include <linux/netdevice.h>
  45 #include <linux/if_bridge.h>
  46 #include <linux/socket.h>
  47 #include <linux/route.h>
  48 #include <net/netevent.h>
  49 #include <net/neighbour.h>
  50 #include <net/arp.h>
  51 #include <net/ip_fib.h>
  52 #include <net/ip6_fib.h>
  53 #include <net/fib_rules.h>
  54 #include <net/l3mdev.h>
  55 #include <net/addrconf.h>
  56 #include <net/ndisc.h>
  57 #include <net/ipv6.h>
  58 #include <net/fib_notifier.h>
  59
  60 #include "spectrum.h"
  61 #include "core.h"
  62 #include "reg.h"
  63 #include "spectrum_cnt.h"
  64 #include "spectrum_dpipe.h"
  65 #include "spectrum_router.h"
  66
  67 struct mlxsw_sp_vr;
  68 struct mlxsw_sp_lpm_tree;
  69 struct mlxsw_sp_rif_ops;
  70
  71 struct mlxsw_sp_router {
  72         struct mlxsw_sp *mlxsw_sp;
  73         struct mlxsw_sp_rif **rifs;
  74         struct mlxsw_sp_vr *vrs;
  75         struct rhashtable neigh_ht;
  76         struct rhashtable nexthop_group_ht;
  77         struct rhashtable nexthop_ht;
  78         struct {
  79                 struct mlxsw_sp_lpm_tree *trees;
  80                 unsigned int tree_count;
  81         } lpm;
  82         struct {
  83                 struct delayed_work dw;
  84                 unsigned long interval; /* ms */
  85         } neighs_update;
  86         struct delayed_work nexthop_probe_dw;
  87 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  88         struct list_head nexthop_neighs_list;
  89         bool aborted;
  90         struct notifier_block fib_nb;
  91         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  92 };
  93
  94 struct mlxsw_sp_rif {
  95         struct list_head nexthop_list;
  96         struct list_head neigh_list;
  97         struct net_device *dev;
  98         struct mlxsw_sp_fid *fid;
  99         unsigned char addr[ETH_ALEN];
 100         int mtu;
 101         u16 rif_index;
 102         u16 vr_id;
 103         const struct mlxsw_sp_rif_ops *ops;
 104         struct mlxsw_sp *mlxsw_sp;
 105
 106         unsigned int counter_ingress;
 107         bool counter_ingress_valid;
 108         unsigned int counter_egress;
 109         bool counter_egress_valid;
 110 };
 111
 112 struct mlxsw_sp_rif_params {
 113         struct net_device *dev;
 114         union {
 115                 u16 system_port;
 116                 u16 lag_id;
 117         };
 118         u16 vid;
 119         bool lag;
 120 };
 121
 122 struct mlxsw_sp_rif_subport {
 123         struct mlxsw_sp_rif common;
 124         union {
 125                 u16 system_port;
 126                 u16 lag_id;
 127         };
 128         u16 vid;
 129         bool lag;
 130 };
 131
 132 struct mlxsw_sp_rif_ops {
 133         enum mlxsw_sp_rif_type type;
 134         size_t rif_size;
 135
 136         void (*setup)(struct mlxsw_sp_rif *rif,
 137                       const struct mlxsw_sp_rif_params *params);
 138         int (*configure)(struct mlxsw_sp_rif *rif);
 139         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 140         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 141 };
 142
 143 static unsigned int *
 144 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 145                            enum mlxsw_sp_rif_counter_dir dir)
 146 {
 147         switch (dir) {
 148         case MLXSW_SP_RIF_COUNTER_EGRESS:
 149                 return &rif->counter_egress;
 150         case MLXSW_SP_RIF_COUNTER_INGRESS:
 151                 return &rif->counter_ingress;
 152         }
 153         return NULL;
 154 }
 155
 156 static bool
 157 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 158                                enum mlxsw_sp_rif_counter_dir dir)
 159 {
 160         switch (dir) {
 161         case MLXSW_SP_RIF_COUNTER_EGRESS:
 162                 return rif->counter_egress_valid;
 163         case MLXSW_SP_RIF_COUNTER_INGRESS:
 164                 return rif->counter_ingress_valid;
 165         }
 166         return false;
 167 }
 168
 169 static void
 170 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 171                                enum mlxsw_sp_rif_counter_dir dir,
 172                                bool valid)
 173 {
 174         switch (dir) {
 175         case MLXSW_SP_RIF_COUNTER_EGRESS:
 176                 rif->counter_egress_valid = valid;
 177                 break;
 178         case MLXSW_SP_RIF_COUNTER_INGRESS:
 179                 rif->counter_ingress_valid = valid;
 180                 break;
 181         }
 182 }
 183
 184 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 185                                      unsigned int counter_index, bool enable,
 186                                      enum mlxsw_sp_rif_counter_dir dir)
 187 {
 188         char ritr_pl[MLXSW_REG_RITR_LEN];
 189         bool is_egress = false;
 190         int err;
 191
 192         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 193                 is_egress = true;
 194         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 195         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 196         if (err)
 197                 return err;
 198
 199         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 200                                     is_egress);
 201         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 202 }
 203
 204 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 205                                    struct mlxsw_sp_rif *rif,
 206                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 207 {
 208         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 209         unsigned int *p_counter_index;
 210         bool valid;
 211         int err;
 212
 213         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 214         if (!valid)
 215                 return -EINVAL;
 216
 217         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 218         if (!p_counter_index)
 219                 return -EINVAL;
 220         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 221                              MLXSW_REG_RICNT_OPCODE_NOP);
 222         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 223         if (err)
 224                 return err;
 225         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 226         return 0;
 227 }
 228
 229 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 230                                       unsigned int counter_index)
 231 {
 232         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 233
 234         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 235                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 236         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 237 }
 238
 239 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 240                                struct mlxsw_sp_rif *rif,
 241                                enum mlxsw_sp_rif_counter_dir dir)
 242 {
 243         unsigned int *p_counter_index;
 244         int err;
 245
 246         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 247         if (!p_counter_index)
 248                 return -EINVAL;
 249         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 250                                      p_counter_index);
 251         if (err)
 252                 return err;
 253
 254         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 255         if (err)
 256                 goto err_counter_clear;
 257
 258         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 259                                         *p_counter_index, true, dir);
 260         if (err)
 261                 goto err_counter_edit;
 262         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 263         return 0;
 264
 265 err_counter_edit:
 266 err_counter_clear:
 267         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 268                               *p_counter_index);
 269         return err;
 270 }
 271
 272 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 273                                struct mlxsw_sp_rif *rif,
 274                                enum mlxsw_sp_rif_counter_dir dir)
 275 {
 276         unsigned int *p_counter_index;
 277
 278         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 279                 return;
 280
 281         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 282         if (WARN_ON(!p_counter_index))
 283                 return;
 284         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 285                                   *p_counter_index, false, dir);
 286         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 287                               *p_counter_index);
 288         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 289 }
 290
 291 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 292 {
 293         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 294         struct devlink *devlink;
 295
 296         devlink = priv_to_devlink(mlxsw_sp->core);
 297         if (!devlink_dpipe_table_counter_enabled(devlink,
 298                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 299                 return;
 300         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 301 }
 302
 303 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 304 {
 305         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 306
 307         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 308 }
 309
 310 static struct mlxsw_sp_rif *
 311 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 312                          const struct net_device *dev);
 313
 314 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 315
 316 struct mlxsw_sp_prefix_usage {
 317         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 318 };
 319
 320 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 321         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 322
 323 static bool
 324 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 325                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 326 {
 327         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 328 }
 329
 330 static bool
 331 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
 332 {
 333         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
 334
 335         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 336 }
 337
 338 static void
 339 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 340                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 341 {
 342         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 343 }
 344
 345 static void
 346 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 347                           unsigned char prefix_len)
 348 {
 349         set_bit(prefix_len, prefix_usage->b);
 350 }
 351
 352 static void
 353 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 354                             unsigned char prefix_len)
 355 {
 356         clear_bit(prefix_len, prefix_usage->b);
 357 }
 358
 359 struct mlxsw_sp_fib_key {
 360         unsigned char addr[sizeof(struct in6_addr)];
 361         unsigned char prefix_len;
 362 };
 363
 364 enum mlxsw_sp_fib_entry_type {
 365         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 366         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 367         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 368 };
 369
 370 struct mlxsw_sp_nexthop_group;
 371 struct mlxsw_sp_fib;
 372
 373 struct mlxsw_sp_fib_node {
 374         struct list_head entry_list;
 375         struct list_head list;
 376         struct rhash_head ht_node;
 377         struct mlxsw_sp_fib *fib;
 378         struct mlxsw_sp_fib_key key;
 379 };
 380
 381 struct mlxsw_sp_fib_entry {
 382         struct list_head list;
 383         struct mlxsw_sp_fib_node *fib_node;
 384         enum mlxsw_sp_fib_entry_type type;
 385         struct list_head nexthop_group_node;
 386         struct mlxsw_sp_nexthop_group *nh_group;
 387 };
 388
 389 struct mlxsw_sp_fib4_entry {
 390         struct mlxsw_sp_fib_entry common;
 391         u32 tb_id;
 392         u32 prio;
 393         u8 tos;
 394         u8 type;
 395 };
 396
 397 struct mlxsw_sp_fib6_entry {
 398         struct mlxsw_sp_fib_entry common;
 399         struct list_head rt6_list;
 400         unsigned int nrt6;
 401 };
 402
 403 struct mlxsw_sp_rt6 {
 404         struct list_head list;
 405         struct rt6_info *rt;
 406 };
 407
 408 struct mlxsw_sp_lpm_tree {
 409         u8 id; /* tree ID */
 410         unsigned int ref_count;
 411         enum mlxsw_sp_l3proto proto;
 412         struct mlxsw_sp_prefix_usage prefix_usage;
 413 };
 414
 415 struct mlxsw_sp_fib {
 416         struct rhashtable ht;
 417         struct list_head node_list;
 418         struct mlxsw_sp_vr *vr;
 419         struct mlxsw_sp_lpm_tree *lpm_tree;
 420         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 421         struct mlxsw_sp_prefix_usage prefix_usage;
 422         enum mlxsw_sp_l3proto proto;
 423 };
 424
 425 struct mlxsw_sp_vr {
 426         u16 id; /* virtual router ID */
 427         u32 tb_id; /* kernel fib table id */
 428         unsigned int rif_count;
 429         struct mlxsw_sp_fib *fib4;
 430         struct mlxsw_sp_fib *fib6;
 431 };
 432
 433 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 434
 435 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 436                                                 enum mlxsw_sp_l3proto proto)
 437 {
 438         struct mlxsw_sp_fib *fib;
 439         int err;
 440
 441         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 442         if (!fib)
 443                 return ERR_PTR(-ENOMEM);
 444         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 445         if (err)
 446                 goto err_rhashtable_init;
 447         INIT_LIST_HEAD(&fib->node_list);
 448         fib->proto = proto;
 449         fib->vr = vr;
 450         return fib;
 451
 452 err_rhashtable_init:
 453         kfree(fib);
 454         return ERR_PTR(err);
 455 }
 456
 457 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 458 {
 459         WARN_ON(!list_empty(&fib->node_list));
 460         WARN_ON(fib->lpm_tree);
 461         rhashtable_destroy(&fib->ht);
 462         kfree(fib);
 463 }
 464
 465 static struct mlxsw_sp_lpm_tree *
 466 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 467 {
 468         static struct mlxsw_sp_lpm_tree *lpm_tree;
 469         int i;
 470
 471         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 472                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 473                 if (lpm_tree->ref_count == 0)
 474                         return lpm_tree;
 475         }
 476         return NULL;
 477 }
 478
 479 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 480                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 481 {
 482         char ralta_pl[MLXSW_REG_RALTA_LEN];
 483
 484         mlxsw_reg_ralta_pack(ralta_pl, true,
 485                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 486                              lpm_tree->id);
 487         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 488 }
 489
 490 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 491                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 492 {
 493         char ralta_pl[MLXSW_REG_RALTA_LEN];
 494
 495         mlxsw_reg_ralta_pack(ralta_pl, false,
 496                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 497                              lpm_tree->id);
 498         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 499 }
 500
 501 static int
 502 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 503                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 504                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 505 {
 506         char ralst_pl[MLXSW_REG_RALST_LEN];
 507         u8 root_bin = 0;
 508         u8 prefix;
 509         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 510
 511         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 512                 root_bin = prefix;
 513
 514         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 515         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 516                 if (prefix == 0)
 517                         continue;
 518                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 519                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 520                 last_prefix = prefix;
 521         }
 522         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 523 }
 524
 525 static struct mlxsw_sp_lpm_tree *
 526 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 527                          struct mlxsw_sp_prefix_usage *prefix_usage,
 528                          enum mlxsw_sp_l3proto proto)
 529 {
 530         struct mlxsw_sp_lpm_tree *lpm_tree;
 531         int err;
 532
 533         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 534         if (!lpm_tree)
 535                 return ERR_PTR(-EBUSY);
 536         lpm_tree->proto = proto;
 537         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 538         if (err)
 539                 return ERR_PTR(err);
 540
 541         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 542                                                 lpm_tree);
 543         if (err)
 544                 goto err_left_struct_set;
 545         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 546                sizeof(lpm_tree->prefix_usage));
 547         return lpm_tree;
 548
 549 err_left_struct_set:
 550         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 551         return ERR_PTR(err);
 552 }
 553
 554 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 555                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 556 {
 557         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 558 }
 559
 560 static struct mlxsw_sp_lpm_tree *
 561 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 562                       struct mlxsw_sp_prefix_usage *prefix_usage,
 563                       enum mlxsw_sp_l3proto proto)
 564 {
 565         struct mlxsw_sp_lpm_tree *lpm_tree;
 566         int i;
 567
 568         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 569                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 570                 if (lpm_tree->ref_count != 0 &&
 571                     lpm_tree->proto == proto &&
 572                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 573                                              prefix_usage))
 574                         return lpm_tree;
 575         }
 576         return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
 577 }
 578
 579 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 580 {
 581         lpm_tree->ref_count++;
 582 }
 583
 584 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 585                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 586 {
 587         if (--lpm_tree->ref_count == 0)
 588                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 589 }
 590
 591 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 592
 593 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 594 {
 595         struct mlxsw_sp_lpm_tree *lpm_tree;
 596         u64 max_trees;
 597         int i;
 598
 599         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 600                 return -EIO;
 601
 602         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 603         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 604         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 605                                              sizeof(struct mlxsw_sp_lpm_tree),
 606                                              GFP_KERNEL);
 607         if (!mlxsw_sp->router->lpm.trees)
 608                 return -ENOMEM;
 609
 610         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 611                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 612                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 613         }
 614
 615         return 0;
 616 }
 617
 618 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 619 {
 620         kfree(mlxsw_sp->router->lpm.trees);
 621 }
 622
 623 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 624 {
 625         return !!vr->fib4 || !!vr->fib6;
 626 }
 627
 628 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 629 {
 630         struct mlxsw_sp_vr *vr;
 631         int i;
 632
 633         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 634                 vr = &mlxsw_sp->router->vrs[i];
 635                 if (!mlxsw_sp_vr_is_used(vr))
 636                         return vr;
 637         }
 638         return NULL;
 639 }
 640
 641 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 642                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 643 {
 644         char raltb_pl[MLXSW_REG_RALTB_LEN];
 645
 646         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 647                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 648                              tree_id);
 649         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 650 }
 651
 652 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 653                                        const struct mlxsw_sp_fib *fib)
 654 {
 655         char raltb_pl[MLXSW_REG_RALTB_LEN];
 656
 657         /* Bind to tree 0 which is default */
 658         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 659                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 660         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 661 }
 662
 663 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 664 {
 665         /* For our purpose, squash main and local table into one */
 666         if (tb_id == RT_TABLE_LOCAL)
 667                 tb_id = RT_TABLE_MAIN;
 668         return tb_id;
 669 }
 670
 671 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 672                                             u32 tb_id)
 673 {
 674         struct mlxsw_sp_vr *vr;
 675         int i;
 676
 677         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 678
 679         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 680                 vr = &mlxsw_sp->router->vrs[i];
 681                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 682                         return vr;
 683         }
 684         return NULL;
 685 }
 686
 687 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 688                                             enum mlxsw_sp_l3proto proto)
 689 {
 690         switch (proto) {
 691         case MLXSW_SP_L3_PROTO_IPV4:
 692                 return vr->fib4;
 693         case MLXSW_SP_L3_PROTO_IPV6:
 694                 return vr->fib6;
 695         }
 696         return NULL;
 697 }
 698
 699 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 700                                               u32 tb_id)
 701 {
 702         struct mlxsw_sp_vr *vr;
 703         int err;
 704
 705         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 706         if (!vr)
 707                 return ERR_PTR(-EBUSY);
 708         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 709         if (IS_ERR(vr->fib4))
 710                 return ERR_CAST(vr->fib4);
 711         vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
 712         if (IS_ERR(vr->fib6)) {
 713                 err = PTR_ERR(vr->fib6);
 714                 goto err_fib6_create;
 715         }
 716         vr->tb_id = tb_id;
 717         return vr;
 718
 719 err_fib6_create:
 720         mlxsw_sp_fib_destroy(vr->fib4);
 721         vr->fib4 = NULL;
 722         return ERR_PTR(err);
 723 }
 724
 725 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 726 {
 727         mlxsw_sp_fib_destroy(vr->fib6);
 728         vr->fib6 = NULL;
 729         mlxsw_sp_fib_destroy(vr->fib4);
 730         vr->fib4 = NULL;
 731 }
 732
 733 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 734 {
 735         struct mlxsw_sp_vr *vr;
 736
 737         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 738         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 739         if (!vr)
 740                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 741         return vr;
 742 }
 743
 744 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 745 {
 746         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 747             list_empty(&vr->fib6->node_list))
 748                 mlxsw_sp_vr_destroy(vr);
 749 }
 750
 751 static bool
 752 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 753                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 754 {
 755         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 756
 757         if (!mlxsw_sp_vr_is_used(vr))
 758                 return false;
 759         if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
 760                 return true;
 761         return false;
 762 }
 763
 764 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 765                                         struct mlxsw_sp_fib *fib,
 766                                         struct mlxsw_sp_lpm_tree *new_tree)
 767 {
 768         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 769         int err;
 770
 771         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 772         if (err)
 773                 return err;
 774         fib->lpm_tree = new_tree;
 775         mlxsw_sp_lpm_tree_hold(new_tree);
 776         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 777         return 0;
 778 }
 779
 780 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 781                                          struct mlxsw_sp_fib *fib,
 782                                          struct mlxsw_sp_lpm_tree *new_tree)
 783 {
 784         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 785         enum mlxsw_sp_l3proto proto = fib->proto;
 786         u8 old_id, new_id = new_tree->id;
 787         struct mlxsw_sp_vr *vr;
 788         int i, err;
 789
 790         if (!old_tree)
 791                 goto no_replace;
 792         old_id = old_tree->id;
 793
 794         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 795                 vr = &mlxsw_sp->router->vrs[i];
 796                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 797                         continue;
 798                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 799                                                    mlxsw_sp_vr_fib(vr, proto),
 800                                                    new_tree);
 801                 if (err)
 802                         goto err_tree_replace;
 803         }
 804
 805         return 0;
 806
 807 err_tree_replace:
 808         for (i--; i >= 0; i--) {
 809                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 810                         continue;
 811                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 812                                              mlxsw_sp_vr_fib(vr, proto),
 813                                              old_tree);
 814         }
 815         return err;
 816
 817 no_replace:
 818         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 819         if (err)
 820                 return err;
 821         fib->lpm_tree = new_tree;
 822         mlxsw_sp_lpm_tree_hold(new_tree);
 823         return 0;
 824 }
 825
 826 static void
 827 mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
 828                       enum mlxsw_sp_l3proto proto,
 829                       struct mlxsw_sp_prefix_usage *req_prefix_usage)
 830 {
 831         int i;
 832
 833         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 834                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
 835                 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 836                 unsigned char prefix;
 837
 838                 if (!mlxsw_sp_vr_is_used(vr))
 839                         continue;
 840                 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
 841                         mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
 842         }
 843 }
 844
 845 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 846 {
 847         struct mlxsw_sp_vr *vr;
 848         u64 max_vrs;
 849         int i;
 850
 851         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 852                 return -EIO;
 853
 854         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 855         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 856                                         GFP_KERNEL);
 857         if (!mlxsw_sp->router->vrs)
 858                 return -ENOMEM;
 859
 860         for (i = 0; i < max_vrs; i++) {
 861                 vr = &mlxsw_sp->router->vrs[i];
 862                 vr->id = i;
 863         }
 864
 865         return 0;
 866 }
 867
 868 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 869
 870 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 871 {
 872         /* At this stage we're guaranteed not to have new incoming
 873          * FIB notifications and the work queue is free from FIBs
 874          * sitting on top of mlxsw netdevs. However, we can still
 875          * have other FIBs queued. Flush the queue before flushing
 876          * the device's tables. No need for locks, as we're the only
 877          * writer.
 878          */
 879         mlxsw_core_flush_owq();
 880         mlxsw_sp_router_fib_flush(mlxsw_sp);
 881         kfree(mlxsw_sp->router->vrs);
 882 }
 883
 884 struct mlxsw_sp_neigh_key {
 885         struct neighbour *n;
 886 };
 887
 888 struct mlxsw_sp_neigh_entry {
 889         struct list_head rif_list_node;
 890         struct rhash_head ht_node;
 891         struct mlxsw_sp_neigh_key key;
 892         u16 rif;
 893         bool connected;
 894         unsigned char ha[ETH_ALEN];
 895         struct list_head nexthop_list; /* list of nexthops using
 896                                         * this neigh entry
 897                                         */
 898         struct list_head nexthop_neighs_list_node;
 899         unsigned int counter_index;
 900         bool counter_valid;
 901 };
 902
 903 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 904         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 905         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 906         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 907 };
 908
 909 struct mlxsw_sp_neigh_entry *
 910 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
 911                         struct mlxsw_sp_neigh_entry *neigh_entry)
 912 {
 913         if (!neigh_entry) {
 914                 if (list_empty(&rif->neigh_list))
 915                         return NULL;
 916                 else
 917                         return list_first_entry(&rif->neigh_list,
 918                                                 typeof(*neigh_entry),
 919                                                 rif_list_node);
 920         }
 921         if (neigh_entry->rif_list_node.next == &rif->neigh_list)
 922                 return NULL;
 923         return list_next_entry(neigh_entry, rif_list_node);
 924 }
 925
 926 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
 927 {
 928         return neigh_entry->key.n->tbl->family;
 929 }
 930
 931 unsigned char *
 932 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
 933 {
 934         return neigh_entry->ha;
 935 }
 936
 937 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
 938 {
 939         struct neighbour *n;
 940
 941         n = neigh_entry->key.n;
 942         return ntohl(*((__be32 *) n->primary_key));
 943 }
 944
 945 struct in6_addr *
 946 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
 947 {
 948         struct neighbour *n;
 949
 950         n = neigh_entry->key.n;
 951         return (struct in6_addr *) &n->primary_key;
 952 }
 953
 954 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
 955                                struct mlxsw_sp_neigh_entry *neigh_entry,
 956                                u64 *p_counter)
 957 {
 958         if (!neigh_entry->counter_valid)
 959                 return -EINVAL;
 960
 961         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
 962                                          p_counter, NULL);
 963 }
 964
 965 static struct mlxsw_sp_neigh_entry *
 966 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 967                            u16 rif)
 968 {
 969         struct mlxsw_sp_neigh_entry *neigh_entry;
 970
 971         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
 972         if (!neigh_entry)
 973                 return NULL;
 974
 975         neigh_entry->key.n = n;
 976         neigh_entry->rif = rif;
 977         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 978
 979         return neigh_entry;
 980 }
 981
 982 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 983 {
 984         kfree(neigh_entry);
 985 }
 986
 987 static int
 988 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 989                             struct mlxsw_sp_neigh_entry *neigh_entry)
 990 {
 991         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
 992                                       &neigh_entry->ht_node,
 993                                       mlxsw_sp_neigh_ht_params);
 994 }
 995
 996 static void
 997 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 998                             struct mlxsw_sp_neigh_entry *neigh_entry)
 999 {
1000         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
1001                                &neigh_entry->ht_node,
1002                                mlxsw_sp_neigh_ht_params);
1003 }
1004
1005 static bool
1006 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
1007                                     struct mlxsw_sp_neigh_entry *neigh_entry)
1008 {
1009         struct devlink *devlink;
1010         const char *table_name;
1011
1012         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
1013         case AF_INET:
1014                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
1015                 break;
1016         case AF_INET6:
1017                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
1018                 break;
1019         default:
1020                 WARN_ON(1);
1021                 return false;
1022         }
1023
1024         devlink = priv_to_devlink(mlxsw_sp->core);
1025         return devlink_dpipe_table_counter_enabled(devlink, table_name);
1026 }
1027
1028 static void
1029 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
1030                              struct mlxsw_sp_neigh_entry *neigh_entry)
1031 {
1032         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
1033                 return;
1034
1035         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
1036                 return;
1037
1038         neigh_entry->counter_valid = true;
1039 }
1040
1041 static void
1042 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
1043                             struct mlxsw_sp_neigh_entry *neigh_entry)
1044 {
1045         if (!neigh_entry->counter_valid)
1046                 return;
1047         mlxsw_sp_flow_counter_free(mlxsw_sp,
1048                                    neigh_entry->counter_index);
1049         neigh_entry->counter_valid = false;
1050 }
1051
1052 static struct mlxsw_sp_neigh_entry *
1053 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1054 {
1055         struct mlxsw_sp_neigh_entry *neigh_entry;
1056         struct mlxsw_sp_rif *rif;
1057         int err;
1058
1059         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
1060         if (!rif)
1061                 return ERR_PTR(-EINVAL);
1062
1063         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
1064         if (!neigh_entry)
1065                 return ERR_PTR(-ENOMEM);
1066
1067         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
1068         if (err)
1069                 goto err_neigh_entry_insert;
1070
1071         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1072         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
1073
1074         return neigh_entry;
1075
1076 err_neigh_entry_insert:
1077         mlxsw_sp_neigh_entry_free(neigh_entry);
1078         return ERR_PTR(err);
1079 }
1080
1081 static void
1082 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1083                              struct mlxsw_sp_neigh_entry *neigh_entry)
1084 {
1085         list_del(&neigh_entry->rif_list_node);
1086         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1087         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
1088         mlxsw_sp_neigh_entry_free(neigh_entry);
1089 }
1090
1091 static struct mlxsw_sp_neigh_entry *
1092 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1093 {
1094         struct mlxsw_sp_neigh_key key;
1095
1096         key.n = n;
1097         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
1098                                       &key, mlxsw_sp_neigh_ht_params);
1099 }
1100
1101 static void
1102 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
1103 {
1104         unsigned long interval;
1105
1106 #if IS_ENABLED(CONFIG_IPV6)
1107         interval = min_t(unsigned long,
1108                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
1109                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
1110 #else
1111         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
1112 #endif
1113         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
1114 }
1115
1116 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1117                                                    char *rauhtd_pl,
1118                                                    int ent_index)
1119 {
1120         struct net_device *dev;
1121         struct neighbour *n;
1122         __be32 dipn;
1123         u32 dip;
1124         u16 rif;
1125
1126         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
1127
1128         if (!mlxsw_sp->router->rifs[rif]) {
1129                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1130                 return;
1131         }
1132
1133         dipn = htonl(dip);
1134         dev = mlxsw_sp->router->rifs[rif]->dev;
1135         n = neigh_lookup(&arp_tbl, &dipn, dev);
1136         if (!n) {
1137                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
1138                            &dip);
1139                 return;
1140         }
1141
1142         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
1143         neigh_event_send(n, NULL);
1144         neigh_release(n);
1145 }
1146
1147 #if IS_ENABLED(CONFIG_IPV6)
1148 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1149                                                    char *rauhtd_pl,
1150                                                    int rec_index)
1151 {
1152         struct net_device *dev;
1153         struct neighbour *n;
1154         struct in6_addr dip;
1155         u16 rif;
1156
1157         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
1158                                          (char *) &dip);
1159
1160         if (!mlxsw_sp->router->rifs[rif]) {
1161                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1162                 return;
1163         }
1164
1165         dev = mlxsw_sp->router->rifs[rif]->dev;
1166         n = neigh_lookup(&nd_tbl, &dip, dev);
1167         if (!n) {
1168                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
1169                            &dip);
1170                 return;
1171         }
1172
1173         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1174         neigh_event_send(n, NULL);
1175         neigh_release(n);
1176 }
1177 #else
1178 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1179                                                    char *rauhtd_pl,
1180                                                    int rec_index)
1181 {
1182 }
1183 #endif
1184
1185 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1186                                                    char *rauhtd_pl,
1187                                                    int rec_index)
1188 {
1189         u8 num_entries;
1190         int i;
1191
1192         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1193                                                                 rec_index);
1194         /* Hardware starts counting at 0, so add 1. */
1195         num_entries++;
1196
1197         /* Each record consists of several neighbour entries. */
1198         for (i = 0; i < num_entries; i++) {
1199                 int ent_index;
1200
1201                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1202                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1203                                                        ent_index);
1204         }
1205
1206 }
1207
1208 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1209                                                    char *rauhtd_pl,
1210                                                    int rec_index)
1211 {
1212         /* One record contains one entry. */
1213         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1214                                                rec_index);
1215 }
1216
1217 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1218                                               char *rauhtd_pl, int rec_index)
1219 {
1220         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1221         case MLXSW_REG_RAUHTD_TYPE_IPV4:
1222                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1223                                                        rec_index);
1224                 break;
1225         case MLXSW_REG_RAUHTD_TYPE_IPV6:
1226                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1227                                                        rec_index);
1228                 break;
1229         }
1230 }
1231
1232 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1233 {
1234         u8 num_rec, last_rec_index, num_entries;
1235
1236         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1237         last_rec_index = num_rec - 1;
1238
1239         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1240                 return false;
1241         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1242             MLXSW_REG_RAUHTD_TYPE_IPV6)
1243                 return true;
1244
1245         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1246                                                                 last_rec_index);
1247         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1248                 return true;
1249         return false;
1250 }
1251
1252 static int
1253 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1254                                        char *rauhtd_pl,
1255                                        enum mlxsw_reg_rauhtd_type type)
1256 {
1257         int i, num_rec;
1258         int err;
1259
1260         /* Make sure the neighbour's netdev isn't removed in the
1261          * process.
1262          */
1263         rtnl_lock();
1264         do {
1265                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
1266                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1267                                       rauhtd_pl);
1268                 if (err) {
1269                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1270                         break;
1271                 }
1272                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1273                 for (i = 0; i < num_rec; i++)
1274                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1275                                                           i);
1276         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1277         rtnl_unlock();
1278
1279         return err;
1280 }
1281
1282 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1283 {
1284         enum mlxsw_reg_rauhtd_type type;
1285         char *rauhtd_pl;
1286         int err;
1287
1288         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1289         if (!rauhtd_pl)
1290                 return -ENOMEM;
1291
1292         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1293         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1294         if (err)
1295                 goto out;
1296
1297         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1298         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1299 out:
1300         kfree(rauhtd_pl);
1301         return err;
1302 }
1303
1304 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1305 {
1306         struct mlxsw_sp_neigh_entry *neigh_entry;
1307
1308         /* Take RTNL mutex here to prevent lists from changes */
1309         rtnl_lock();
1310         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1311                             nexthop_neighs_list_node)
1312                 /* If this neigh have nexthops, make the kernel think this neigh
1313                  * is active regardless of the traffic.
1314                  */
1315                 neigh_event_send(neigh_entry->key.n, NULL);
1316         rtnl_unlock();
1317 }
1318
1319 static void
1320 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1321 {
1322         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1323
1324         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1325                                msecs_to_jiffies(interval));
1326 }
1327
1328 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1329 {
1330         struct mlxsw_sp_router *router;
1331         int err;
1332
1333         router = container_of(work, struct mlxsw_sp_router,
1334                               neighs_update.dw.work);
1335         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1336         if (err)
1337                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1338
1339         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1340
1341         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1342 }
1343
1344 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1345 {
1346         struct mlxsw_sp_neigh_entry *neigh_entry;
1347         struct mlxsw_sp_router *router;
1348
1349         router = container_of(work, struct mlxsw_sp_router,
1350                               nexthop_probe_dw.work);
1351         /* Iterate over nexthop neighbours, find those who are unresolved and
1352          * send arp on them. This solves the chicken-egg problem when
1353          * the nexthop wouldn't get offloaded until the neighbor is resolved
1354          * but it wouldn't get resolved ever in case traffic is flowing in HW
1355          * using different nexthop.
1356          *
1357          * Take RTNL mutex here to prevent lists from changes.
1358          */
1359         rtnl_lock();
1360         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1361                             nexthop_neighs_list_node)
1362                 if (!neigh_entry->connected)
1363                         neigh_event_send(neigh_entry->key.n, NULL);
1364         rtnl_unlock();
1365
1366         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1367                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1368 }
1369
1370 static void
1371 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1372                               struct mlxsw_sp_neigh_entry *neigh_entry,
1373                               bool removing);
1374
1375 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1376 {
1377         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1378                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1379 }
1380
1381 static void
1382 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1383                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1384                                 enum mlxsw_reg_rauht_op op)
1385 {
1386         struct neighbour *n = neigh_entry->key.n;
1387         u32 dip = ntohl(*((__be32 *) n->primary_key));
1388         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1389
1390         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1391                               dip);
1392         if (neigh_entry->counter_valid)
1393                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1394                                              neigh_entry->counter_index);
1395         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1396 }
1397
1398 static void
1399 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1400                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1401                                 enum mlxsw_reg_rauht_op op)
1402 {
1403         struct neighbour *n = neigh_entry->key.n;
1404         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1405         const char *dip = n->primary_key;
1406
1407         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1408                               dip);
1409         if (neigh_entry->counter_valid)
1410                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1411                                              neigh_entry->counter_index);
1412         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1413 }
1414
1415 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
1416 {
1417         struct neighbour *n = neigh_entry->key.n;
1418
1419         /* Packets with a link-local destination address are trapped
1420          * after LPM lookup and never reach the neighbour table, so
1421          * there is no need to program such neighbours to the device.
1422          */
1423         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1424             IPV6_ADDR_LINKLOCAL)
1425                 return true;
1426         return false;
1427 }
1428
1429 static void
1430 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1431                             struct mlxsw_sp_neigh_entry *neigh_entry,
1432                             bool adding)
1433 {
1434         if (!adding && !neigh_entry->connected)
1435                 return;
1436         neigh_entry->connected = adding;
1437         if (neigh_entry->key.n->tbl->family == AF_INET) {
1438                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1439                                                 mlxsw_sp_rauht_op(adding));
1440         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
1441                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
1442                         return;
1443                 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1444                                                 mlxsw_sp_rauht_op(adding));
1445         } else {
1446                 WARN_ON_ONCE(1);
1447         }
1448 }
1449
1450 void
1451 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
1452                                     struct mlxsw_sp_neigh_entry *neigh_entry,
1453                                     bool adding)
1454 {
1455         if (adding)
1456                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1457         else
1458                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1459         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
1460 }
1461
1462 struct mlxsw_sp_neigh_event_work {
1463         struct work_struct work;
1464         struct mlxsw_sp *mlxsw_sp;
1465         struct neighbour *n;
1466 };
1467
1468 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1469 {
1470         struct mlxsw_sp_neigh_event_work *neigh_work =
1471                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1472         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1473         struct mlxsw_sp_neigh_entry *neigh_entry;
1474         struct neighbour *n = neigh_work->n;
1475         unsigned char ha[ETH_ALEN];
1476         bool entry_connected;
1477         u8 nud_state, dead;
1478
1479         /* If these parameters are changed after we release the lock,
1480          * then we are guaranteed to receive another event letting us
1481          * know about it.
1482          */
1483         read_lock_bh(&n->lock);
1484         memcpy(ha, n->ha, ETH_ALEN);
1485         nud_state = n->nud_state;
1486         dead = n->dead;
1487         read_unlock_bh(&n->lock);
1488
1489         rtnl_lock();
1490         entry_connected = nud_state & NUD_VALID && !dead;
1491         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1492         if (!entry_connected && !neigh_entry)
1493                 goto out;
1494         if (!neigh_entry) {
1495                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1496                 if (IS_ERR(neigh_entry))
1497                         goto out;
1498         }
1499
1500         memcpy(neigh_entry->ha, ha, ETH_ALEN);
1501         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1502         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1503
1504         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1505                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1506
1507 out:
1508         rtnl_unlock();
1509         neigh_release(n);
1510         kfree(neigh_work);
1511 }
1512
1513 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1514                                    unsigned long event, void *ptr)
1515 {
1516         struct mlxsw_sp_neigh_event_work *neigh_work;
1517         struct mlxsw_sp_port *mlxsw_sp_port;
1518         struct mlxsw_sp *mlxsw_sp;
1519         unsigned long interval;
1520         struct neigh_parms *p;
1521         struct neighbour *n;
1522
1523         switch (event) {
1524         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1525                 p = ptr;
1526
1527                 /* We don't care about changes in the default table. */
1528                 if (!p->dev || (p->tbl->family != AF_INET &&
1529                                 p->tbl->family != AF_INET6))
1530                         return NOTIFY_DONE;
1531
1532                 /* We are in atomic context and can't take RTNL mutex,
1533                  * so use RCU variant to walk the device chain.
1534                  */
1535                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1536                 if (!mlxsw_sp_port)
1537                         return NOTIFY_DONE;
1538
1539                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1540                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1541                 mlxsw_sp->router->neighs_update.interval = interval;
1542
1543                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1544                 break;
1545         case NETEVENT_NEIGH_UPDATE:
1546                 n = ptr;
1547
1548                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
1549                         return NOTIFY_DONE;
1550
1551                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1552                 if (!mlxsw_sp_port)
1553                         return NOTIFY_DONE;
1554
1555                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1556                 if (!neigh_work) {
1557                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1558                         return NOTIFY_BAD;
1559                 }
1560
1561                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1562                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1563                 neigh_work->n = n;
1564
1565                 /* Take a reference to ensure the neighbour won't be
1566                  * destructed until we drop the reference in delayed
1567                  * work.
1568                  */
1569                 neigh_clone(n);
1570                 mlxsw_core_schedule_work(&neigh_work->work);
1571                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1572                 break;
1573         }
1574
1575         return NOTIFY_DONE;
1576 }
1577
1578 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1579 {
1580         int err;
1581
1582         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1583                               &mlxsw_sp_neigh_ht_params);
1584         if (err)
1585                 return err;
1586
1587         /* Initialize the polling interval according to the default
1588          * table.
1589          */
1590         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1591
1592         /* Create the delayed works for the activity_update */
1593         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1594                           mlxsw_sp_router_neighs_update_work);
1595         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1596                           mlxsw_sp_router_probe_unresolved_nexthops);
1597         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1598         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1599         return 0;
1600 }
1601
1602 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1603 {
1604         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1605         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1606         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
1607 }
1608
1609 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1610                                          struct mlxsw_sp_rif *rif)
1611 {
1612         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1613
1614         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1615                                  rif_list_node) {
1616                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
1617                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1618         }
1619 }
1620
1621 struct mlxsw_sp_nexthop_key {
1622         struct fib_nh *fib_nh;
1623 };
1624
1625 struct mlxsw_sp_nexthop {
1626         struct list_head neigh_list_node; /* member of neigh entry list */
1627         struct list_head rif_list_node;
1628         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1629                                                 * this belongs to
1630                                                 */
1631         struct rhash_head ht_node;
1632         struct mlxsw_sp_nexthop_key key;
1633         unsigned char gw_addr[sizeof(struct in6_addr)];
1634         int ifindex;
1635         struct mlxsw_sp_rif *rif;
1636         u8 should_offload:1, /* set indicates this neigh is connected and
1637                               * should be put to KVD linear area of this group.
1638                               */
1639            offloaded:1, /* set in case the neigh is actually put into
1640                          * KVD linear area of this group.
1641                          */
1642            update:1; /* set indicates that MAC of this neigh should be
1643                       * updated in HW
1644                       */
1645         struct mlxsw_sp_neigh_entry *neigh_entry;
1646 };
1647
1648 struct mlxsw_sp_nexthop_group {
1649         void *priv;
1650         struct rhash_head ht_node;
1651         struct list_head fib_list; /* list of fib entries that use this group */
1652         struct neigh_table *neigh_tbl;
1653         u8 adj_index_valid:1,
1654            gateway:1; /* routes using the group use a gateway */
1655         u32 adj_index;
1656         u16 ecmp_size;
1657         u16 count;
1658         struct mlxsw_sp_nexthop nexthops[0];
1659 #define nh_rif  nexthops[0].rif
1660 };
1661
1662 static struct fib_info *
1663 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
1664 {
1665         return nh_grp->priv;
1666 }
1667
1668 struct mlxsw_sp_nexthop_group_cmp_arg {
1669         enum mlxsw_sp_l3proto proto;
1670         union {
1671                 struct fib_info *fi;
1672                 struct mlxsw_sp_fib6_entry *fib6_entry;
1673         };
1674 };
1675
1676 static bool
1677 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
1678                                     const struct in6_addr *gw, int ifindex)
1679 {
1680         int i;
1681
1682         for (i = 0; i < nh_grp->count; i++) {
1683                 const struct mlxsw_sp_nexthop *nh;
1684
1685                 nh = &nh_grp->nexthops[i];
1686                 if (nh->ifindex == ifindex &&
1687                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
1688                         return true;
1689         }
1690
1691         return false;
1692 }
1693
1694 static bool
1695 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
1696                             const struct mlxsw_sp_fib6_entry *fib6_entry)
1697 {
1698         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
1699
1700         if (nh_grp->count != fib6_entry->nrt6)
1701                 return false;
1702
1703         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
1704                 struct in6_addr *gw;
1705                 int ifindex;
1706
1707                 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
1708                 gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
1709                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
1710                         return false;
1711         }
1712
1713         return true;
1714 }
1715
1716 static int
1717 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
1718 {
1719         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
1720         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
1721
1722         switch (cmp_arg->proto) {
1723         case MLXSW_SP_L3_PROTO_IPV4:
1724                 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
1725         case MLXSW_SP_L3_PROTO_IPV6:
1726                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
1727                                                     cmp_arg->fib6_entry);
1728         default:
1729                 WARN_ON(1);
1730                 return 1;
1731         }
1732 }
1733
1734 static int
1735 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
1736 {
1737         return nh_grp->neigh_tbl->family;
1738 }
1739
1740 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
1741 {
1742         const struct mlxsw_sp_nexthop_group *nh_grp = data;
1743         const struct mlxsw_sp_nexthop *nh;
1744         struct fib_info *fi;
1745         unsigned int val;
1746         int i;
1747
1748         switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
1749         case AF_INET:
1750                 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
1751                 return jhash(&fi, sizeof(fi), seed);
1752         case AF_INET6:
1753                 val = nh_grp->count;
1754                 for (i = 0; i < nh_grp->count; i++) {
1755                         nh = &nh_grp->nexthops[i];
1756                         val ^= nh->ifindex;
1757                 }
1758                 return jhash(&val, sizeof(val), seed);
1759         default:
1760                 WARN_ON(1);
1761                 return 0;
1762         }
1763 }
1764
1765 static u32
1766 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
1767 {
1768         unsigned int val = fib6_entry->nrt6;
1769         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
1770         struct net_device *dev;
1771
1772         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
1773                 dev = mlxsw_sp_rt6->rt->dst.dev;
1774                 val ^= dev->ifindex;
1775         }
1776
1777         return jhash(&val, sizeof(val), seed);
1778 }
1779
1780 static u32
1781 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
1782 {
1783         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
1784
1785         switch (cmp_arg->proto) {
1786         case MLXSW_SP_L3_PROTO_IPV4:
1787                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
1788         case MLXSW_SP_L3_PROTO_IPV6:
1789                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
1790         default:
1791                 WARN_ON(1);
1792                 return 0;
1793         }
1794 }
1795
1796 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1797         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1798         .hashfn      = mlxsw_sp_nexthop_group_hash,
1799         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
1800         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
1801 };
1802
1803 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1804                                          struct mlxsw_sp_nexthop_group *nh_grp)
1805 {
1806         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
1807             !nh_grp->gateway)
1808                 return 0;
1809
1810         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
1811                                       &nh_grp->ht_node,
1812                                       mlxsw_sp_nexthop_group_ht_params);
1813 }
1814
1815 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1816                                           struct mlxsw_sp_nexthop_group *nh_grp)
1817 {
1818         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
1819             !nh_grp->gateway)
1820                 return;
1821
1822         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
1823                                &nh_grp->ht_node,
1824                                mlxsw_sp_nexthop_group_ht_params);
1825 }
1826
1827 static struct mlxsw_sp_nexthop_group *
1828 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
1829                                struct fib_info *fi)
1830 {
1831         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
1832
1833         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
1834         cmp_arg.fi = fi;
1835         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
1836                                       &cmp_arg,
1837                                       mlxsw_sp_nexthop_group_ht_params);
1838 }
1839
1840 static struct mlxsw_sp_nexthop_group *
1841 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
1842                                struct mlxsw_sp_fib6_entry *fib6_entry)
1843 {
1844         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
1845
1846         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
1847         cmp_arg.fib6_entry = fib6_entry;
1848         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
1849                                       &cmp_arg,
1850                                       mlxsw_sp_nexthop_group_ht_params);
1851 }
1852
1853 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1854         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1855         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1856         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1857 };
1858
1859 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1860                                    struct mlxsw_sp_nexthop *nh)
1861 {
1862         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
1863                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1864 }
1865
1866 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1867                                     struct mlxsw_sp_nexthop *nh)
1868 {
1869         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
1870                                mlxsw_sp_nexthop_ht_params);
1871 }
1872
1873 static struct mlxsw_sp_nexthop *
1874 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1875                         struct mlxsw_sp_nexthop_key key)
1876 {
1877         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
1878                                       mlxsw_sp_nexthop_ht_params);
1879 }
1880
1881 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1882                                              const struct mlxsw_sp_fib *fib,
1883                                              u32 adj_index, u16 ecmp_size,
1884                                              u32 new_adj_index,
1885                                              u16 new_ecmp_size)
1886 {
1887         char raleu_pl[MLXSW_REG_RALEU_LEN];
1888
1889         mlxsw_reg_raleu_pack(raleu_pl,
1890                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
1891                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
1892                              new_ecmp_size);
1893         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1894 }
1895
1896 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1897                                           struct mlxsw_sp_nexthop_group *nh_grp,
1898                                           u32 old_adj_index, u16 old_ecmp_size)
1899 {
1900         struct mlxsw_sp_fib_entry *fib_entry;
1901         struct mlxsw_sp_fib *fib = NULL;
1902         int err;
1903
1904         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1905                 if (fib == fib_entry->fib_node->fib)
1906                         continue;
1907                 fib = fib_entry->fib_node->fib;
1908                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1909                                                         old_adj_index,
1910                                                         old_ecmp_size,
1911                                                         nh_grp->adj_index,
1912                                                         nh_grp->ecmp_size);
1913                 if (err)
1914                         return err;
1915         }
1916         return 0;
1917 }
1918
1919 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1920                                        struct mlxsw_sp_nexthop *nh)
1921 {
1922         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1923         char ratr_pl[MLXSW_REG_RATR_LEN];
1924
1925         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1926                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
1927                             adj_index, neigh_entry->rif);
1928         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1929         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1930 }
1931
1932 static int
1933 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1934                                   struct mlxsw_sp_nexthop_group *nh_grp,
1935                                   bool reallocate)
1936 {
1937         u32 adj_index = nh_grp->adj_index; /* base */
1938         struct mlxsw_sp_nexthop *nh;
1939         int i;
1940         int err;
1941
1942         for (i = 0; i < nh_grp->count; i++) {
1943                 nh = &nh_grp->nexthops[i];
1944
1945                 if (!nh->should_offload) {
1946                         nh->offloaded = 0;
1947                         continue;
1948                 }
1949
1950                 if (nh->update || reallocate) {
1951                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1952                                                           adj_index, nh);
1953                         if (err)
1954                                 return err;
1955                         nh->update = 0;
1956                         nh->offloaded = 1;
1957                 }
1958                 adj_index++;
1959         }
1960         return 0;
1961 }
1962
1963 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1964                                      struct mlxsw_sp_fib_entry *fib_entry);
1965
1966 static bool
1967 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
1968                                  const struct mlxsw_sp_fib_entry *fib_entry);
1969
1970 static int
1971 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1972                                     struct mlxsw_sp_nexthop_group *nh_grp)
1973 {
1974         struct mlxsw_sp_fib_entry *fib_entry;
1975         int err;
1976
1977         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1978                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
1979                                                       fib_entry))
1980                         continue;
1981                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1982                 if (err)
1983                         return err;
1984         }
1985         return 0;
1986 }
1987
1988 static void
1989 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1990                                    enum mlxsw_reg_ralue_op op, int err);
1991
1992 static void
1993 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
1994 {
1995         enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
1996         struct mlxsw_sp_fib_entry *fib_entry;
1997
1998         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1999                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2000                                                       fib_entry))
2001                         continue;
2002                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2003         }
2004 }
2005
2006 static void
2007 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
2008                                struct mlxsw_sp_nexthop_group *nh_grp)
2009 {
2010         struct mlxsw_sp_nexthop *nh;
2011         bool offload_change = false;
2012         u32 adj_index;
2013         u16 ecmp_size = 0;
2014         bool old_adj_index_valid;
2015         u32 old_adj_index;
2016         u16 old_ecmp_size;
2017         int i;
2018         int err;
2019
2020         if (!nh_grp->gateway) {
2021                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2022                 return;
2023         }
2024
2025         for (i = 0; i < nh_grp->count; i++) {
2026                 nh = &nh_grp->nexthops[i];
2027
2028                 if (nh->should_offload != nh->offloaded) {
2029                         offload_change = true;
2030                         if (nh->should_offload)
2031                                 nh->update = 1;
2032                 }
2033                 if (nh->should_offload)
2034                         ecmp_size++;
2035         }
2036         if (!offload_change) {
2037                 /* Nothing was added or removed, so no need to reallocate. Just
2038                  * update MAC on existing adjacency indexes.
2039                  */
2040                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
2041                                                         false);
2042                 if (err) {
2043                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2044                         goto set_trap;
2045                 }
2046                 return;
2047         }
2048         if (!ecmp_size)
2049                 /* No neigh of this group is connected so we just set
2050                  * the trap and let everthing flow through kernel.
2051                  */
2052                 goto set_trap;
2053
2054         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
2055         if (err) {
2056                 /* We ran out of KVD linear space, just set the
2057                  * trap and let everything flow through kernel.
2058                  */
2059                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
2060                 goto set_trap;
2061         }
2062         old_adj_index_valid = nh_grp->adj_index_valid;
2063         old_adj_index = nh_grp->adj_index;
2064         old_ecmp_size = nh_grp->ecmp_size;
2065         nh_grp->adj_index_valid = 1;
2066         nh_grp->adj_index = adj_index;
2067         nh_grp->ecmp_size = ecmp_size;
2068         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
2069         if (err) {
2070                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2071                 goto set_trap;
2072         }
2073
2074         if (!old_adj_index_valid) {
2075                 /* The trap was set for fib entries, so we have to call
2076                  * fib entry update to unset it and use adjacency index.
2077                  */
2078                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2079                 if (err) {
2080                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
2081                         goto set_trap;
2082                 }
2083                 return;
2084         }
2085
2086         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
2087                                              old_adj_index, old_ecmp_size);
2088         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
2089         if (err) {
2090                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
2091                 goto set_trap;
2092         }
2093
2094         /* Offload state within the group changed, so update the flags. */
2095         mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
2096
2097         return;
2098
2099 set_trap:
2100         old_adj_index_valid = nh_grp->adj_index_valid;
2101         nh_grp->adj_index_valid = 0;
2102         for (i = 0; i < nh_grp->count; i++) {
2103                 nh = &nh_grp->nexthops[i];
2104                 nh->offloaded = 0;
2105         }
2106         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2107         if (err)
2108                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
2109         if (old_adj_index_valid)
2110                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
2111 }
2112
2113 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
2114                                             bool removing)
2115 {
2116         if (!removing)
2117                 nh->should_offload = 1;
2118         else if (nh->offloaded)
2119                 nh->should_offload = 0;
2120         nh->update = 1;
2121 }
2122
2123 static void
2124 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2125                               struct mlxsw_sp_neigh_entry *neigh_entry,
2126                               bool removing)
2127 {
2128         struct mlxsw_sp_nexthop *nh;
2129
2130         list_for_each_entry(nh, &neigh_entry->nexthop_list,
2131                             neigh_list_node) {
2132                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
2133                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2134         }
2135 }
2136
2137 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
2138                                       struct mlxsw_sp_rif *rif)
2139 {
2140         if (nh->rif)
2141                 return;
2142
2143         nh->rif = rif;
2144         list_add(&nh->rif_list_node, &rif->nexthop_list);
2145 }
2146
2147 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
2148 {
2149         if (!nh->rif)
2150                 return;
2151
2152         list_del(&nh->rif_list_node);
2153         nh->rif = NULL;
2154 }
2155
2156 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
2157                                        struct mlxsw_sp_nexthop *nh)
2158 {
2159         struct mlxsw_sp_neigh_entry *neigh_entry;
2160         struct neighbour *n;
2161         u8 nud_state, dead;
2162         int err;
2163
2164         if (!nh->nh_grp->gateway || nh->neigh_entry)
2165                 return 0;
2166
2167         /* Take a reference of neigh here ensuring that neigh would
2168          * not be destructed before the nexthop entry is finished.
2169          * The reference is taken either in neigh_lookup() or
2170          * in neigh_create() in case n is not found.
2171          */
2172         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
2173         if (!n) {
2174                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
2175                                  nh->rif->dev);
2176                 if (IS_ERR(n))
2177                         return PTR_ERR(n);
2178                 neigh_event_send(n, NULL);
2179         }
2180         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2181         if (!neigh_entry) {
2182                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2183                 if (IS_ERR(neigh_entry)) {
2184                         err = -EINVAL;
2185                         goto err_neigh_entry_create;
2186                 }
2187         }
2188
2189         /* If that is the first nexthop connected to that neigh, add to
2190          * nexthop_neighs_list
2191          */
2192         if (list_empty(&neigh_entry->nexthop_list))
2193                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
2194                               &mlxsw_sp->router->nexthop_neighs_list);
2195
2196         nh->neigh_entry = neigh_entry;
2197         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
2198         read_lock_bh(&n->lock);
2199         nud_state = n->nud_state;
2200         dead = n->dead;
2201         read_unlock_bh(&n->lock);
2202         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
2203
2204         return 0;
2205
2206 err_neigh_entry_create:
2207         neigh_release(n);
2208         return err;
2209 }
2210
2211 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
2212                                         struct mlxsw_sp_nexthop *nh)
2213 {
2214         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2215         struct neighbour *n;
2216
2217         if (!neigh_entry)
2218                 return;
2219         n = neigh_entry->key.n;
2220
2221         __mlxsw_sp_nexthop_neigh_update(nh, true);
2222         list_del(&nh->neigh_list_node);
2223         nh->neigh_entry = NULL;
2224
2225         /* If that is the last nexthop connected to that neigh, remove from
2226          * nexthop_neighs_list
2227          */
2228         if (list_empty(&neigh_entry->nexthop_list))
2229                 list_del(&neigh_entry->nexthop_neighs_list_node);
2230
2231         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2232                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2233
2234         neigh_release(n);
2235 }
2236
2237 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
2238                                   struct mlxsw_sp_nexthop_group *nh_grp,
2239                                   struct mlxsw_sp_nexthop *nh,
2240                                   struct fib_nh *fib_nh)
2241 {
2242         struct net_device *dev = fib_nh->nh_dev;
2243         struct in_device *in_dev;
2244         struct mlxsw_sp_rif *rif;
2245         int err;
2246
2247         nh->nh_grp = nh_grp;
2248         nh->key.fib_nh = fib_nh;
2249         memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
2250         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
2251         if (err)
2252                 return err;
2253
2254         if (!dev)
2255                 return 0;
2256
2257         in_dev = __in_dev_get_rtnl(dev);
2258         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
2259             fib_nh->nh_flags & RTNH_F_LINKDOWN)
2260                 return 0;
2261
2262         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
2263         if (!rif)
2264                 return 0;
2265         mlxsw_sp_nexthop_rif_init(nh, rif);
2266
2267         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2268         if (err)
2269                 goto err_nexthop_neigh_init;
2270
2271         return 0;
2272
2273 err_nexthop_neigh_init:
2274         mlxsw_sp_nexthop_rif_fini(nh);
2275         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2276         return err;
2277 }
2278
2279 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
2280                                    struct mlxsw_sp_nexthop *nh)
2281 {
2282         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2283         mlxsw_sp_nexthop_rif_fini(nh);
2284         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2285 }
2286
2287 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
2288                                     unsigned long event, struct fib_nh *fib_nh)
2289 {
2290         struct mlxsw_sp_nexthop_key key;
2291         struct mlxsw_sp_nexthop *nh;
2292         struct mlxsw_sp_rif *rif;
2293
2294         if (mlxsw_sp->router->aborted)
2295                 return;
2296
2297         key.fib_nh = fib_nh;
2298         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
2299         if (WARN_ON_ONCE(!nh))
2300                 return;
2301
2302         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
2303         if (!rif)
2304                 return;
2305
2306         switch (event) {
2307         case FIB_EVENT_NH_ADD:
2308                 mlxsw_sp_nexthop_rif_init(nh, rif);
2309                 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2310                 break;
2311         case FIB_EVENT_NH_DEL:
2312                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2313                 mlxsw_sp_nexthop_rif_fini(nh);
2314                 break;
2315         }
2316
2317         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2318 }
2319
2320 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2321                                            struct mlxsw_sp_rif *rif)
2322 {
2323         struct mlxsw_sp_nexthop *nh, *tmp;
2324
2325         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
2326                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2327                 mlxsw_sp_nexthop_rif_fini(nh);
2328                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2329         }
2330 }
2331
2332 static struct mlxsw_sp_nexthop_group *
2333 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
2334 {
2335         struct mlxsw_sp_nexthop_group *nh_grp;
2336         struct mlxsw_sp_nexthop *nh;
2337         struct fib_nh *fib_nh;
2338         size_t alloc_size;
2339         int i;
2340         int err;
2341
2342         alloc_size = sizeof(*nh_grp) +
2343                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
2344         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
2345         if (!nh_grp)
2346                 return ERR_PTR(-ENOMEM);
2347         nh_grp->priv = fi;
2348         INIT_LIST_HEAD(&nh_grp->fib_list);
2349         nh_grp->neigh_tbl = &arp_tbl;
2350
2351         nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
2352         nh_grp->count = fi->fib_nhs;
2353         fib_info_hold(fi);
2354         for (i = 0; i < nh_grp->count; i++) {
2355                 nh = &nh_grp->nexthops[i];
2356                 fib_nh = &fi->fib_nh[i];
2357                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
2358                 if (err)
2359                         goto err_nexthop4_init;
2360         }
2361         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
2362         if (err)
2363                 goto err_nexthop_group_insert;
2364         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2365         return nh_grp;
2366
2367 err_nexthop_group_insert:
2368 err_nexthop4_init:
2369         for (i--; i >= 0; i--) {
2370                 nh = &nh_grp->nexthops[i];
2371                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2372         }
2373         fib_info_put(fi);
2374         kfree(nh_grp);
2375         return ERR_PTR(err);
2376 }
2377
2378 static void
2379 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
2380                                 struct mlxsw_sp_nexthop_group *nh_grp)
2381 {
2382         struct mlxsw_sp_nexthop *nh;
2383         int i;
2384
2385         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
2386         for (i = 0; i < nh_grp->count; i++) {
2387                 nh = &nh_grp->nexthops[i];
2388                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2389         }
2390         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2391         WARN_ON_ONCE(nh_grp->adj_index_valid);
2392         fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
2393         kfree(nh_grp);
2394 }
2395
2396 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
2397                                        struct mlxsw_sp_fib_entry *fib_entry,
2398                                        struct fib_info *fi)
2399 {
2400         struct mlxsw_sp_nexthop_group *nh_grp;
2401
2402         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
2403         if (!nh_grp) {
2404                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
2405                 if (IS_ERR(nh_grp))
2406                         return PTR_ERR(nh_grp);
2407         }
2408         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
2409         fib_entry->nh_group = nh_grp;
2410         return 0;
2411 }
2412
2413 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
2414                                         struct mlxsw_sp_fib_entry *fib_entry)
2415 {
2416         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2417
2418         list_del(&fib_entry->nexthop_group_node);
2419         if (!list_empty(&nh_grp->fib_list))
2420                 return;
2421         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
2422 }
2423
2424 static bool
2425 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2426 {
2427         struct mlxsw_sp_fib4_entry *fib4_entry;
2428
2429         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
2430                                   common);
2431         return !fib4_entry->tos;
2432 }
2433
2434 static bool
2435 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2436 {
2437         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
2438
2439         switch (fib_entry->fib_node->fib->proto) {
2440         case MLXSW_SP_L3_PROTO_IPV4:
2441                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
2442                         return false;
2443                 break;
2444         case MLXSW_SP_L3_PROTO_IPV6:
2445                 break;
2446         }
2447
2448         switch (fib_entry->type) {
2449         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2450                 return !!nh_group->adj_index_valid;
2451         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2452                 return !!nh_group->nh_rif;
2453         default:
2454                 return false;
2455         }
2456 }
2457
2458 static struct mlxsw_sp_nexthop *
2459 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
2460                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
2461 {
2462         int i;
2463
2464         for (i = 0; i < nh_grp->count; i++) {
2465                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2466                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2467
2468                 if (nh->rif && nh->rif->dev == rt->dst.dev &&
2469                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
2470                                     &rt->rt6i_gateway))
2471                         return nh;
2472                 continue;
2473         }
2474
2475         return NULL;
2476 }
2477
2478 static void
2479 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2480 {
2481         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2482         int i;
2483
2484         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
2485                 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
2486                 return;
2487         }
2488
2489         for (i = 0; i < nh_grp->count; i++) {
2490                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2491
2492                 if (nh->offloaded)
2493                         nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
2494                 else
2495                         nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
2496         }
2497 }
2498
2499 static void
2500 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2501 {
2502         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2503         int i;
2504
2505         for (i = 0; i < nh_grp->count; i++) {
2506                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2507
2508                 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
2509         }
2510 }
2511
2512 static void
2513 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2514 {
2515         struct mlxsw_sp_fib6_entry *fib6_entry;
2516         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2517
2518         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
2519                                   common);
2520
2521         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
2522                 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
2523                                  list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
2524                 return;
2525         }
2526
2527         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2528                 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2529                 struct mlxsw_sp_nexthop *nh;
2530
2531                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
2532                 if (nh && nh->offloaded)
2533                         mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
2534                 else
2535                         mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
2536         }
2537 }
2538
2539 static void
2540 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2541 {
2542         struct mlxsw_sp_fib6_entry *fib6_entry;
2543         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2544
2545         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
2546                                   common);
2547         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2548                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2549
2550                 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
2551         }
2552 }
2553
2554 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2555 {
2556         switch (fib_entry->fib_node->fib->proto) {
2557         case MLXSW_SP_L3_PROTO_IPV4:
2558                 mlxsw_sp_fib4_entry_offload_set(fib_entry);
2559                 break;
2560         case MLXSW_SP_L3_PROTO_IPV6:
2561                 mlxsw_sp_fib6_entry_offload_set(fib_entry);
2562                 break;
2563         }
2564 }
2565
2566 static void
2567 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2568 {
2569         switch (fib_entry->fib_node->fib->proto) {
2570         case MLXSW_SP_L3_PROTO_IPV4:
2571                 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
2572                 break;
2573         case MLXSW_SP_L3_PROTO_IPV6:
2574                 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
2575                 break;
2576         }
2577 }
2578
2579 static void
2580 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2581                                    enum mlxsw_reg_ralue_op op, int err)
2582 {
2583         switch (op) {
2584         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
2585                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
2586         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
2587                 if (err)
2588                         return;
2589                 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
2590                         mlxsw_sp_fib_entry_offload_set(fib_entry);
2591                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
2592                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
2593                 return;
2594         default:
2595                 return;
2596         }
2597 }
2598
2599 static void
2600 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
2601                               const struct mlxsw_sp_fib_entry *fib_entry,
2602                               enum mlxsw_reg_ralue_op op)
2603 {
2604         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2605         enum mlxsw_reg_ralxx_protocol proto;
2606         u32 *p_dip;
2607
2608         proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
2609
2610         switch (fib->proto) {
2611         case MLXSW_SP_L3_PROTO_IPV4:
2612                 p_dip = (u32 *) fib_entry->fib_node->key.addr;
2613                 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
2614                                       fib_entry->fib_node->key.prefix_len,
2615                                       *p_dip);
2616                 break;
2617         case MLXSW_SP_L3_PROTO_IPV6:
2618                 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
2619                                       fib_entry->fib_node->key.prefix_len,
2620                                       fib_entry->fib_node->key.addr);
2621                 break;
2622         }
2623 }
2624
2625 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
2626                                         struct mlxsw_sp_fib_entry *fib_entry,
2627                                         enum mlxsw_reg_ralue_op op)
2628 {
2629         char ralue_pl[MLXSW_REG_RALUE_LEN];
2630         enum mlxsw_reg_ralue_trap_action trap_action;
2631         u16 trap_id = 0;
2632         u32 adjacency_index = 0;
2633         u16 ecmp_size = 0;
2634
2635         /* In case the nexthop group adjacency index is valid, use it
2636          * with provided ECMP size. Otherwise, setup trap and pass
2637          * traffic to kernel.
2638          */
2639         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2640                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2641                 adjacency_index = fib_entry->nh_group->adj_index;
2642                 ecmp_size = fib_entry->nh_group->ecmp_size;
2643         } else {
2644                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2645                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2646         }
2647
2648         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
2649         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2650                                         adjacency_index, ecmp_size);
2651         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2652 }
2653
2654 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
2655                                        struct mlxsw_sp_fib_entry *fib_entry,
2656                                        enum mlxsw_reg_ralue_op op)
2657 {
2658         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
2659         enum mlxsw_reg_ralue_trap_action trap_action;
2660         char ralue_pl[MLXSW_REG_RALUE_LEN];
2661         u16 trap_id = 0;
2662         u16 rif_index = 0;
2663
2664         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2665                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2666                 rif_index = rif->rif_index;
2667         } else {
2668                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2669                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2670         }
2671
2672         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
2673         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
2674                                        rif_index);
2675         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2676 }
2677
2678 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
2679                                       struct mlxsw_sp_fib_entry *fib_entry,
2680                                       enum mlxsw_reg_ralue_op op)
2681 {
2682         char ralue_pl[MLXSW_REG_RALUE_LEN];
2683
2684         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
2685         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2686         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2687 }
2688
2689 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2690                                    struct mlxsw_sp_fib_entry *fib_entry,
2691                                    enum mlxsw_reg_ralue_op op)
2692 {
2693         switch (fib_entry->type) {
2694         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2695                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
2696         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2697                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
2698         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
2699                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
2700         }
2701         return -EINVAL;
2702 }
2703
2704 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2705                                  struct mlxsw_sp_fib_entry *fib_entry,
2706                                  enum mlxsw_reg_ralue_op op)
2707 {
2708         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
2709
2710         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
2711
2712         return err;
2713 }
2714
2715 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2716                                      struct mlxsw_sp_fib_entry *fib_entry)
2717 {
2718         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2719                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
2720 }
2721
2722 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
2723                                   struct mlxsw_sp_fib_entry *fib_entry)
2724 {
2725         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2726                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
2727 }
2728
2729 static int
2730 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
2731                              const struct fib_entry_notifier_info *fen_info,
2732                              struct mlxsw_sp_fib_entry *fib_entry)
2733 {
2734         struct fib_info *fi = fen_info->fi;
2735
2736         switch (fen_info->type) {
2737         case RTN_BROADCAST: /* fall through */
2738         case RTN_LOCAL:
2739                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2740                 return 0;
2741         case RTN_UNREACHABLE: /* fall through */
2742         case RTN_BLACKHOLE: /* fall through */
2743         case RTN_PROHIBIT:
2744                 /* Packets hitting these routes need to be trapped, but
2745                  * can do so with a lower priority than packets directed
2746                  * at the host, so use action type local instead of trap.
2747                  */
2748                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2749                 return 0;
2750         case RTN_UNICAST:
2751                 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
2752                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2753                 else
2754                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
2755                 return 0;
2756         default:
2757                 return -EINVAL;
2758         }
2759 }
2760
2761 static struct mlxsw_sp_fib4_entry *
2762 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
2763                            struct mlxsw_sp_fib_node *fib_node,
2764                            const struct fib_entry_notifier_info *fen_info)
2765 {
2766         struct mlxsw_sp_fib4_entry *fib4_entry;
2767         struct mlxsw_sp_fib_entry *fib_entry;
2768         int err;
2769
2770         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
2771         if (!fib4_entry)
2772                 return ERR_PTR(-ENOMEM);
2773         fib_entry = &fib4_entry->common;
2774
2775         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
2776         if (err)
2777                 goto err_fib4_entry_type_set;
2778
2779         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
2780         if (err)
2781                 goto err_nexthop4_group_get;
2782
2783         fib4_entry->prio = fen_info->fi->fib_priority;
2784         fib4_entry->tb_id = fen_info->tb_id;
2785         fib4_entry->type = fen_info->type;
2786         fib4_entry->tos = fen_info->tos;
2787
2788         fib_entry->fib_node = fib_node;
2789
2790         return fib4_entry;
2791
2792 err_nexthop4_group_get:
2793 err_fib4_entry_type_set:
2794         kfree(fib4_entry);
2795         return ERR_PTR(err);
2796 }
2797
2798 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2799                                         struct mlxsw_sp_fib4_entry *fib4_entry)
2800 {
2801         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
2802         kfree(fib4_entry);
2803 }
2804
2805 static struct mlxsw_sp_fib_node *
2806 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2807                          size_t addr_len, unsigned char prefix_len);
2808
2809 static struct mlxsw_sp_fib4_entry *
2810 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
2811                            const struct fib_entry_notifier_info *fen_info)
2812 {
2813         struct mlxsw_sp_fib4_entry *fib4_entry;
2814         struct mlxsw_sp_fib_node *fib_node;
2815         struct mlxsw_sp_fib *fib;
2816         struct mlxsw_sp_vr *vr;
2817
2818         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
2819         if (!vr)
2820                 return NULL;
2821         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2822
2823         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2824                                             sizeof(fen_info->dst),
2825                                             fen_info->dst_len);
2826         if (!fib_node)
2827                 return NULL;
2828
2829         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
2830                 if (fib4_entry->tb_id == fen_info->tb_id &&
2831                     fib4_entry->tos == fen_info->tos &&
2832                     fib4_entry->type == fen_info->type &&
2833                     mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
2834                     fen_info->fi) {
2835                         return fib4_entry;
2836                 }
2837         }
2838
2839         return NULL;
2840 }
2841
2842 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
2843         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
2844         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
2845         .key_len = sizeof(struct mlxsw_sp_fib_key),
2846         .automatic_shrinking = true,
2847 };
2848
2849 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
2850                                     struct mlxsw_sp_fib_node *fib_node)
2851 {
2852         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
2853                                       mlxsw_sp_fib_ht_params);
2854 }
2855
2856 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
2857                                      struct mlxsw_sp_fib_node *fib_node)
2858 {
2859         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
2860                                mlxsw_sp_fib_ht_params);
2861 }
2862
2863 static struct mlxsw_sp_fib_node *
2864 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2865                          size_t addr_len, unsigned char prefix_len)
2866 {
2867         struct mlxsw_sp_fib_key key;
2868
2869         memset(&key, 0, sizeof(key));
2870         memcpy(key.addr, addr, addr_len);
2871         key.prefix_len = prefix_len;
2872         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2873 }
2874
2875 static struct mlxsw_sp_fib_node *
2876 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
2877                          size_t addr_len, unsigned char prefix_len)
2878 {
2879         struct mlxsw_sp_fib_node *fib_node;
2880
2881         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2882         if (!fib_node)
2883                 return NULL;
2884
2885         INIT_LIST_HEAD(&fib_node->entry_list);
2886         list_add(&fib_node->list, &fib->node_list);
2887         memcpy(fib_node->key.addr, addr, addr_len);
2888         fib_node->key.prefix_len = prefix_len;
2889
2890         return fib_node;
2891 }
2892
2893 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2894 {
2895         list_del(&fib_node->list);
2896         WARN_ON(!list_empty(&fib_node->entry_list));
2897         kfree(fib_node);
2898 }
2899
2900 static bool
2901 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2902                                  const struct mlxsw_sp_fib_entry *fib_entry)
2903 {
2904         return list_first_entry(&fib_node->entry_list,
2905                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
2906 }
2907
2908 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
2909                                       struct mlxsw_sp_fib *fib,
2910                                       struct mlxsw_sp_fib_node *fib_node)
2911 {
2912         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
2913         struct mlxsw_sp_lpm_tree *lpm_tree;
2914         int err;
2915
2916         /* Since the tree is shared between all virtual routers we must
2917          * make sure it contains all the required prefix lengths. This
2918          * can be computed by either adding the new prefix length to the
2919          * existing prefix usage of a bound tree, or by aggregating the
2920          * prefix lengths across all virtual routers and adding the new
2921          * one as well.
2922          */
2923         if (fib->lpm_tree)
2924                 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
2925                                           &fib->lpm_tree->prefix_usage);
2926         else
2927                 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
2928         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2929
2930         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2931                                          fib->proto);
2932         if (IS_ERR(lpm_tree))
2933                 return PTR_ERR(lpm_tree);
2934
2935         if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
2936                 return 0;
2937
2938         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
2939         if (err)
2940                 return err;
2941
2942         return 0;
2943 }
2944
2945 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
2946                                          struct mlxsw_sp_fib *fib)
2947 {
2948         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
2949         struct mlxsw_sp_lpm_tree *lpm_tree;
2950
2951         /* Aggregate prefix lengths across all virtual routers to make
2952          * sure we only have used prefix lengths in the LPM tree.
2953          */
2954         mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
2955         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2956                                          fib->proto);
2957         if (IS_ERR(lpm_tree))
2958                 goto err_tree_get;
2959         mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
2960
2961 err_tree_get:
2962         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
2963                 return;
2964         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2965         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
2966         fib->lpm_tree = NULL;
2967 }
2968
2969 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2970 {
2971         unsigned char prefix_len = fib_node->key.prefix_len;
2972         struct mlxsw_sp_fib *fib = fib_node->fib;
2973
2974         if (fib->prefix_ref_count[prefix_len]++ == 0)
2975                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2976 }
2977
2978 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2979 {
2980         unsigned char prefix_len = fib_node->key.prefix_len;
2981         struct mlxsw_sp_fib *fib = fib_node->fib;
2982
2983         if (--fib->prefix_ref_count[prefix_len] == 0)
2984                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2985 }
2986
2987 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2988                                   struct mlxsw_sp_fib_node *fib_node,
2989                                   struct mlxsw_sp_fib *fib)
2990 {
2991         int err;
2992
2993         err = mlxsw_sp_fib_node_insert(fib, fib_node);
2994         if (err)
2995                 return err;
2996         fib_node->fib = fib;
2997
2998         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
2999         if (err)
3000                 goto err_fib_lpm_tree_link;
3001
3002         mlxsw_sp_fib_node_prefix_inc(fib_node);
3003
3004         return 0;
3005
3006 err_fib_lpm_tree_link:
3007         fib_node->fib = NULL;
3008         mlxsw_sp_fib_node_remove(fib, fib_node);
3009         return err;
3010 }
3011
3012 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
3013                                    struct mlxsw_sp_fib_node *fib_node)
3014 {
3015         struct mlxsw_sp_fib *fib = fib_node->fib;
3016
3017         mlxsw_sp_fib_node_prefix_dec(fib_node);
3018         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
3019         fib_node->fib = NULL;
3020         mlxsw_sp_fib_node_remove(fib, fib_node);
3021 }
3022
3023 static struct mlxsw_sp_fib_node *
3024 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
3025                       size_t addr_len, unsigned char prefix_len,
3026                       enum mlxsw_sp_l3proto proto)
3027 {
3028         struct mlxsw_sp_fib_node *fib_node;
3029         struct mlxsw_sp_fib *fib;
3030         struct mlxsw_sp_vr *vr;
3031         int err;
3032
3033         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
3034         if (IS_ERR(vr))
3035                 return ERR_CAST(vr);
3036         fib = mlxsw_sp_vr_fib(vr, proto);
3037
3038         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
3039         if (fib_node)
3040                 return fib_node;
3041
3042         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
3043         if (!fib_node) {
3044                 err = -ENOMEM;
3045                 goto err_fib_node_create;
3046         }
3047
3048         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
3049         if (err)
3050                 goto err_fib_node_init;
3051
3052         return fib_node;
3053
3054 err_fib_node_init:
3055         mlxsw_sp_fib_node_destroy(fib_node);
3056 err_fib_node_create:
3057         mlxsw_sp_vr_put(vr);
3058         return ERR_PTR(err);
3059 }
3060
3061 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
3062                                   struct mlxsw_sp_fib_node *fib_node)
3063 {
3064         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
3065
3066         if (!list_empty(&fib_node->entry_list))
3067                 return;
3068         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
3069         mlxsw_sp_fib_node_destroy(fib_node);
3070         mlxsw_sp_vr_put(vr);
3071 }
3072
3073 static struct mlxsw_sp_fib4_entry *
3074 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3075                               const struct mlxsw_sp_fib4_entry *new4_entry)
3076 {
3077         struct mlxsw_sp_fib4_entry *fib4_entry;
3078
3079         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3080                 if (fib4_entry->tb_id > new4_entry->tb_id)
3081                         continue;
3082                 if (fib4_entry->tb_id != new4_entry->tb_id)
3083                         break;
3084                 if (fib4_entry->tos > new4_entry->tos)
3085                         continue;
3086                 if (fib4_entry->prio >= new4_entry->prio ||
3087                     fib4_entry->tos < new4_entry->tos)
3088                         return fib4_entry;
3089         }
3090
3091         return NULL;
3092 }
3093
3094 static int
3095 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
3096                                struct mlxsw_sp_fib4_entry *new4_entry)
3097 {
3098         struct mlxsw_sp_fib_node *fib_node;
3099
3100         if (WARN_ON(!fib4_entry))
3101                 return -EINVAL;
3102
3103         fib_node = fib4_entry->common.fib_node;
3104         list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
3105                                  common.list) {
3106                 if (fib4_entry->tb_id != new4_entry->tb_id ||
3107                     fib4_entry->tos != new4_entry->tos ||
3108                     fib4_entry->prio != new4_entry->prio)
3109                         break;
3110         }
3111
3112         list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
3113         return 0;
3114 }
3115
3116 static int
3117 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
3118                                bool replace, bool append)
3119 {
3120         struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
3121         struct mlxsw_sp_fib4_entry *fib4_entry;
3122
3123         fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
3124
3125         if (append)
3126                 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
3127         if (replace && WARN_ON(!fib4_entry))
3128                 return -EINVAL;
3129
3130         /* Insert new entry before replaced one, so that we can later
3131          * remove the second.
3132          */
3133         if (fib4_entry) {
3134                 list_add_tail(&new4_entry->common.list,
3135                               &fib4_entry->common.list);
3136         } else {
3137                 struct mlxsw_sp_fib4_entry *last;
3138
3139                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3140                         if (new4_entry->tb_id > last->tb_id)
3141                                 break;
3142                         fib4_entry = last;
3143                 }
3144
3145                 if (fib4_entry)
3146                         list_add(&new4_entry->common.list,
3147                                  &fib4_entry->common.list);
3148                 else
3149                         list_add(&new4_entry->common.list,
3150                                  &fib_node->entry_list);
3151         }
3152
3153         return 0;
3154 }
3155
3156 static void
3157 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
3158 {
3159         list_del(&fib4_entry->common.list);
3160 }
3161
3162 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
3163                                        struct mlxsw_sp_fib_entry *fib_entry)
3164 {
3165         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3166
3167         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3168                 return 0;
3169
3170         /* To prevent packet loss, overwrite the previously offloaded
3171          * entry.
3172          */
3173         if (!list_is_singular(&fib_node->entry_list)) {
3174                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3175                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3176
3177                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
3178         }
3179
3180         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3181 }
3182
3183 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
3184                                         struct mlxsw_sp_fib_entry *fib_entry)
3185 {
3186         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3187
3188         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3189                 return;
3190
3191         /* Promote the next entry by overwriting the deleted entry */
3192         if (!list_is_singular(&fib_node->entry_list)) {
3193                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3194                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3195
3196                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
3197                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3198                 return;
3199         }
3200
3201         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
3202 }
3203
3204 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3205                                          struct mlxsw_sp_fib4_entry *fib4_entry,
3206                                          bool replace, bool append)
3207 {
3208         int err;
3209
3210         err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
3211         if (err)
3212                 return err;
3213
3214         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
3215         if (err)
3216                 goto err_fib_node_entry_add;
3217
3218         return 0;
3219
3220 err_fib_node_entry_add:
3221         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3222         return err;
3223 }
3224
3225 static void
3226 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3227                                 struct mlxsw_sp_fib4_entry *fib4_entry)
3228 {
3229         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
3230         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3231 }
3232
3233 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
3234                                         struct mlxsw_sp_fib4_entry *fib4_entry,
3235                                         bool replace)
3236 {
3237         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
3238         struct mlxsw_sp_fib4_entry *replaced;
3239
3240         if (!replace)
3241                 return;
3242
3243         /* We inserted the new entry before replaced one */
3244         replaced = list_next_entry(fib4_entry, common.list);
3245
3246         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
3247         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
3248         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3249 }
3250
3251 static int
3252 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
3253                          const struct fib_entry_notifier_info *fen_info,
3254                          bool replace, bool append)
3255 {
3256         struct mlxsw_sp_fib4_entry *fib4_entry;
3257         struct mlxsw_sp_fib_node *fib_node;
3258         int err;
3259
3260         if (mlxsw_sp->router->aborted)
3261                 return 0;
3262
3263         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
3264                                          &fen_info->dst, sizeof(fen_info->dst),
3265                                          fen_info->dst_len,
3266                                          MLXSW_SP_L3_PROTO_IPV4);
3267         if (IS_ERR(fib_node)) {
3268                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
3269                 return PTR_ERR(fib_node);
3270         }
3271
3272         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
3273         if (IS_ERR(fib4_entry)) {
3274                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
3275                 err = PTR_ERR(fib4_entry);
3276                 goto err_fib4_entry_create;
3277         }
3278
3279         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
3280                                             append);
3281         if (err) {
3282                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
3283                 goto err_fib4_node_entry_link;
3284         }
3285
3286         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
3287
3288         return 0;
3289
3290 err_fib4_node_entry_link:
3291         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3292 err_fib4_entry_create:
3293         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3294         return err;
3295 }
3296
3297 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
3298                                      struct fib_entry_notifier_info *fen_info)
3299 {
3300         struct mlxsw_sp_fib4_entry *fib4_entry;
3301         struct mlxsw_sp_fib_node *fib_node;
3302
3303         if (mlxsw_sp->router->aborted)
3304                 return;
3305
3306         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
3307         if (WARN_ON(!fib4_entry))
3308                 return;
3309         fib_node = fib4_entry->common.fib_node;
3310
3311         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
3312         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3313         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3314 }
3315
3316 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
3317 {
3318         /* Packets with link-local destination IP arriving to the router
3319          * are trapped to the CPU, so no need to program specific routes
3320          * for them.
3321          */
3322         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
3323                 return true;
3324
3325         /* Multicast routes aren't supported, so ignore them. Neighbour
3326          * Discovery packets are specifically trapped.
3327          */
3328         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
3329                 return true;
3330
3331         /* Cloned routes are irrelevant in the forwarding path. */
3332         if (rt->rt6i_flags & RTF_CACHE)
3333                 return true;
3334
3335         return false;
3336 }
3337
3338 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
3339 {
3340         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3341
3342         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
3343         if (!mlxsw_sp_rt6)
3344                 return ERR_PTR(-ENOMEM);
3345
3346         /* In case of route replace, replaced route is deleted with
3347          * no notification. Take reference to prevent accessing freed
3348          * memory.
3349          */
3350         mlxsw_sp_rt6->rt = rt;
3351         rt6_hold(rt);
3352
3353         return mlxsw_sp_rt6;
3354 }
3355
3356 #if IS_ENABLED(CONFIG_IPV6)
3357 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3358 {
3359         rt6_release(rt);
3360 }
3361 #else
3362 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3363 {
3364 }
3365 #endif
3366
3367 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3368 {
3369         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
3370         kfree(mlxsw_sp_rt6);
3371 }
3372
3373 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
3374 {
3375         /* RTF_CACHE routes are ignored */
3376         return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
3377 }
3378
3379 static struct rt6_info *
3380 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
3381 {
3382         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3383                                 list)->rt;
3384 }
3385
3386 static struct mlxsw_sp_fib6_entry *
3387 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3388                                  const struct rt6_info *nrt, bool replace)
3389 {
3390         struct mlxsw_sp_fib6_entry *fib6_entry;
3391
3392         if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
3393                 return NULL;
3394
3395         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3396                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3397
3398                 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
3399                  * virtual router.
3400                  */
3401                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3402                         continue;
3403                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3404                         break;
3405                 if (rt->rt6i_metric < nrt->rt6i_metric)
3406                         continue;
3407                 if (rt->rt6i_metric == nrt->rt6i_metric &&
3408                     mlxsw_sp_fib6_rt_can_mp(rt))
3409                         return fib6_entry;
3410                 if (rt->rt6i_metric > nrt->rt6i_metric)
3411                         break;
3412         }
3413
3414         return NULL;
3415 }
3416
3417 static struct mlxsw_sp_rt6 *
3418 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
3419                             const struct rt6_info *rt)
3420 {
3421         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3422
3423         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3424                 if (mlxsw_sp_rt6->rt == rt)
3425                         return mlxsw_sp_rt6;
3426         }
3427
3428         return NULL;
3429 }
3430
3431 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
3432                                   struct mlxsw_sp_nexthop_group *nh_grp,
3433                                   struct mlxsw_sp_nexthop *nh,
3434                                   const struct rt6_info *rt)
3435 {
3436         struct net_device *dev = rt->dst.dev;
3437         struct mlxsw_sp_rif *rif;
3438         int err;
3439
3440         nh->nh_grp = nh_grp;
3441         memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
3442
3443         if (!dev)
3444                 return 0;
3445         nh->ifindex = dev->ifindex;
3446
3447         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3448         if (!rif)
3449                 return 0;
3450         mlxsw_sp_nexthop_rif_init(nh, rif);
3451
3452         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3453         if (err)
3454                 goto err_nexthop_neigh_init;
3455
3456         return 0;
3457
3458 err_nexthop_neigh_init:
3459         mlxsw_sp_nexthop_rif_fini(nh);
3460         return err;
3461 }
3462
3463 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
3464                                    struct mlxsw_sp_nexthop *nh)
3465 {
3466         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
3467         mlxsw_sp_nexthop_rif_fini(nh);
3468 }
3469
3470 static struct mlxsw_sp_nexthop_group *
3471 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
3472                                struct mlxsw_sp_fib6_entry *fib6_entry)
3473 {
3474         struct mlxsw_sp_nexthop_group *nh_grp;
3475         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3476         struct mlxsw_sp_nexthop *nh;
3477         size_t alloc_size;
3478         int i = 0;
3479         int err;
3480
3481         alloc_size = sizeof(*nh_grp) +
3482                      fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
3483         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
3484         if (!nh_grp)
3485                 return ERR_PTR(-ENOMEM);
3486         INIT_LIST_HEAD(&nh_grp->fib_list);
3487 #if IS_ENABLED(CONFIG_IPV6)
3488         nh_grp->neigh_tbl = &nd_tbl;
3489 #endif
3490         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
3491                                         struct mlxsw_sp_rt6, list);
3492         nh_grp->gateway = !!(mlxsw_sp_rt6->rt->rt6i_flags & RTF_GATEWAY);
3493         nh_grp->count = fib6_entry->nrt6;
3494         for (i = 0; i < nh_grp->count; i++) {
3495                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3496
3497                 nh = &nh_grp->nexthops[i];
3498                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
3499                 if (err)
3500                         goto err_nexthop6_init;
3501                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
3502         }
3503
3504         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
3505         if (err)
3506                 goto err_nexthop_group_insert;
3507
3508         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3509         return nh_grp;
3510
3511 err_nexthop_group_insert:
3512 err_nexthop6_init:
3513         for (i--; i >= 0; i--) {
3514                 nh = &nh_grp->nexthops[i];
3515                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
3516         }
3517         kfree(nh_grp);
3518         return ERR_PTR(err);
3519 }
3520
3521 static void
3522 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
3523                                 struct mlxsw_sp_nexthop_group *nh_grp)
3524 {
3525         struct mlxsw_sp_nexthop *nh;
3526         int i = nh_grp->count;
3527
3528         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
3529         for (i--; i >= 0; i--) {
3530                 nh = &nh_grp->nexthops[i];
3531                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
3532         }
3533         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3534         WARN_ON(nh_grp->adj_index_valid);
3535         kfree(nh_grp);
3536 }
3537
3538 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
3539                                        struct mlxsw_sp_fib6_entry *fib6_entry)
3540 {
3541         struct mlxsw_sp_nexthop_group *nh_grp;
3542
3543         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
3544         if (!nh_grp) {
3545                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
3546                 if (IS_ERR(nh_grp))
3547                         return PTR_ERR(nh_grp);
3548         }
3549
3550         list_add_tail(&fib6_entry->common.nexthop_group_node,
3551                       &nh_grp->fib_list);
3552         fib6_entry->common.nh_group = nh_grp;
3553
3554         return 0;
3555 }
3556
3557 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
3558                                         struct mlxsw_sp_fib_entry *fib_entry)
3559 {
3560         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3561
3562         list_del(&fib_entry->nexthop_group_node);
3563         if (!list_empty(&nh_grp->fib_list))
3564                 return;
3565         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
3566 }
3567
3568 static int
3569 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
3570                                struct mlxsw_sp_fib6_entry *fib6_entry)
3571 {
3572         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
3573         int err;
3574
3575         fib6_entry->common.nh_group = NULL;
3576         list_del(&fib6_entry->common.nexthop_group_node);
3577
3578         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
3579         if (err)
3580                 goto err_nexthop6_group_get;
3581
3582         /* In case this entry is offloaded, then the adjacency index
3583          * currently associated with it in the device's table is that
3584          * of the old group. Start using the new one instead.
3585          */
3586         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
3587         if (err)
3588                 goto err_fib_node_entry_add;
3589
3590         if (list_empty(&old_nh_grp->fib_list))
3591                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
3592
3593         return 0;
3594
3595 err_fib_node_entry_add:
3596         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
3597 err_nexthop6_group_get:
3598         list_add_tail(&fib6_entry->common.nexthop_group_node,
3599                       &old_nh_grp->fib_list);
3600         fib6_entry->common.nh_group = old_nh_grp;
3601         return err;
3602 }
3603
3604 static int
3605 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
3606                                 struct mlxsw_sp_fib6_entry *fib6_entry,
3607                                 struct rt6_info *rt)
3608 {
3609         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3610         int err;
3611
3612         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
3613         if (IS_ERR(mlxsw_sp_rt6))
3614                 return PTR_ERR(mlxsw_sp_rt6);
3615
3616         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
3617         fib6_entry->nrt6++;
3618
3619         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
3620         if (err)
3621                 goto err_nexthop6_group_update;
3622
3623         return 0;
3624
3625 err_nexthop6_group_update:
3626         fib6_entry->nrt6--;
3627         list_del(&mlxsw_sp_rt6->list);
3628         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3629         return err;
3630 }
3631
3632 static void
3633 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
3634                                 struct mlxsw_sp_fib6_entry *fib6_entry,
3635                                 struct rt6_info *rt)
3636 {
3637         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3638
3639         mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
3640         if (WARN_ON(!mlxsw_sp_rt6))
3641                 return;
3642
3643         fib6_entry->nrt6--;
3644         list_del(&mlxsw_sp_rt6->list);
3645         mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
3646         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3647 }
3648
3649 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp_fib_entry *fib_entry,
3650                                          const struct rt6_info *rt)
3651 {
3652         /* Packets hitting RTF_REJECT routes need to be discarded by the
3653          * stack. We can rely on their destination device not having a
3654          * RIF (it's the loopback device) and can thus use action type
3655          * local, which will cause them to be trapped with a lower
3656          * priority than packets that need to be locally received.
3657          */
3658         if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
3659                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
3660         else if (rt->rt6i_flags & RTF_REJECT)
3661                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3662         else if (rt->rt6i_flags & RTF_GATEWAY)
3663                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
3664         else
3665                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3666 }
3667
3668 static void
3669 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
3670 {
3671         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
3672
3673         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
3674                                  list) {
3675                 fib6_entry->nrt6--;
3676                 list_del(&mlxsw_sp_rt6->list);
3677                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3678         }
3679 }
3680
3681 static struct mlxsw_sp_fib6_entry *
3682 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
3683                            struct mlxsw_sp_fib_node *fib_node,
3684                            struct rt6_info *rt)
3685 {
3686         struct mlxsw_sp_fib6_entry *fib6_entry;
3687         struct mlxsw_sp_fib_entry *fib_entry;
3688         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3689         int err;
3690
3691         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
3692         if (!fib6_entry)
3693                 return ERR_PTR(-ENOMEM);
3694         fib_entry = &fib6_entry->common;
3695
3696         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
3697         if (IS_ERR(mlxsw_sp_rt6)) {
3698                 err = PTR_ERR(mlxsw_sp_rt6);
3699                 goto err_rt6_create;
3700         }
3701
3702         mlxsw_sp_fib6_entry_type_set(fib_entry, mlxsw_sp_rt6->rt);
3703
3704         INIT_LIST_HEAD(&fib6_entry->rt6_list);
3705         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
3706         fib6_entry->nrt6 = 1;
3707         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
3708         if (err)
3709                 goto err_nexthop6_group_get;
3710
3711         fib_entry->fib_node = fib_node;
3712
3713         return fib6_entry;
3714
3715 err_nexthop6_group_get:
3716         list_del(&mlxsw_sp_rt6->list);
3717         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3718 err_rt6_create:
3719         kfree(fib6_entry);
3720         return ERR_PTR(err);
3721 }
3722
3723 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
3724                                         struct mlxsw_sp_fib6_entry *fib6_entry)
3725 {
3726         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
3727         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
3728         WARN_ON(fib6_entry->nrt6);
3729         kfree(fib6_entry);
3730 }
3731
3732 static struct mlxsw_sp_fib6_entry *
3733 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3734                               const struct rt6_info *nrt, bool replace)
3735 {
3736         struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
3737
3738         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3739                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3740
3741                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3742                         continue;
3743                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3744                         break;
3745                 if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
3746                         if (mlxsw_sp_fib6_rt_can_mp(rt) ==
3747                             mlxsw_sp_fib6_rt_can_mp(nrt))
3748                                 return fib6_entry;
3749                         if (mlxsw_sp_fib6_rt_can_mp(nrt))
3750                                 fallback = fallback ?: fib6_entry;
3751                 }
3752                 if (rt->rt6i_metric > nrt->rt6i_metric)
3753                         return fallback ?: fib6_entry;
3754         }
3755
3756         return fallback;
3757 }
3758
3759 static int
3760 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
3761                                bool replace)
3762 {
3763         struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
3764         struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
3765         struct mlxsw_sp_fib6_entry *fib6_entry;
3766
3767         fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
3768
3769         if (replace && WARN_ON(!fib6_entry))
3770                 return -EINVAL;
3771
3772         if (fib6_entry) {
3773                 list_add_tail(&new6_entry->common.list,
3774                               &fib6_entry->common.list);
3775         } else {
3776                 struct mlxsw_sp_fib6_entry *last;
3777
3778                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3779                         struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
3780
3781                         if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
3782                                 break;
3783                         fib6_entry = last;
3784                 }
3785
3786                 if (fib6_entry)
3787                         list_add(&new6_entry->common.list,
3788                                  &fib6_entry->common.list);
3789                 else
3790                         list_add(&new6_entry->common.list,
3791                                  &fib_node->entry_list);
3792         }
3793
3794         return 0;
3795 }
3796
3797 static void
3798 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
3799 {
3800         list_del(&fib6_entry->common.list);
3801 }
3802
3803 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3804                                          struct mlxsw_sp_fib6_entry *fib6_entry,
3805                                          bool replace)
3806 {
3807         int err;
3808
3809         err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
3810         if (err)
3811                 return err;
3812
3813         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
3814         if (err)
3815                 goto err_fib_node_entry_add;
3816
3817         return 0;
3818
3819 err_fib_node_entry_add:
3820         mlxsw_sp_fib6_node_list_remove(fib6_entry);
3821         return err;
3822 }
3823
3824 static void
3825 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3826                                 struct mlxsw_sp_fib6_entry *fib6_entry)
3827 {
3828         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
3829         mlxsw_sp_fib6_node_list_remove(fib6_entry);
3830 }
3831
3832 static struct mlxsw_sp_fib6_entry *
3833 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
3834                            const struct rt6_info *rt)
3835 {
3836         struct mlxsw_sp_fib6_entry *fib6_entry;
3837         struct mlxsw_sp_fib_node *fib_node;
3838         struct mlxsw_sp_fib *fib;
3839         struct mlxsw_sp_vr *vr;
3840
3841         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
3842         if (!vr)
3843                 return NULL;
3844         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
3845
3846         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
3847                                             sizeof(rt->rt6i_dst.addr),
3848                                             rt->rt6i_dst.plen);
3849         if (!fib_node)
3850                 return NULL;
3851
3852         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3853                 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3854
3855                 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
3856                     rt->rt6i_metric == iter_rt->rt6i_metric &&
3857                     mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
3858                         return fib6_entry;
3859         }
3860
3861         return NULL;
3862 }
3863
3864 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
3865                                         struct mlxsw_sp_fib6_entry *fib6_entry,
3866                                         bool replace)
3867 {
3868         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
3869         struct mlxsw_sp_fib6_entry *replaced;
3870
3871         if (!replace)
3872                 return;
3873
3874         replaced = list_next_entry(fib6_entry, common.list);
3875
3876         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
3877         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
3878         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3879 }
3880
3881 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
3882                                     struct rt6_info *rt, bool replace)
3883 {
3884         struct mlxsw_sp_fib6_entry *fib6_entry;
3885         struct mlxsw_sp_fib_node *fib_node;
3886         int err;
3887
3888         if (mlxsw_sp->router->aborted)
3889                 return 0;
3890
3891         if (rt->rt6i_src.plen)
3892                 return -EINVAL;
3893
3894         if (mlxsw_sp_fib6_rt_should_ignore(rt))
3895                 return 0;
3896
3897         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
3898                                          &rt->rt6i_dst.addr,
3899                                          sizeof(rt->rt6i_dst.addr),
3900                                          rt->rt6i_dst.plen,
3901                                          MLXSW_SP_L3_PROTO_IPV6);
3902         if (IS_ERR(fib_node))
3903                 return PTR_ERR(fib_node);
3904
3905         /* Before creating a new entry, try to append route to an existing
3906          * multipath entry.
3907          */
3908         fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
3909         if (fib6_entry) {
3910                 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
3911                 if (err)
3912                         goto err_fib6_entry_nexthop_add;
3913                 return 0;
3914         }
3915
3916         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
3917         if (IS_ERR(fib6_entry)) {
3918                 err = PTR_ERR(fib6_entry);
3919                 goto err_fib6_entry_create;
3920         }
3921
3922         err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
3923         if (err)
3924                 goto err_fib6_node_entry_link;
3925
3926         mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
3927
3928         return 0;
3929
3930 err_fib6_node_entry_link:
3931         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
3932 err_fib6_entry_create:
3933 err_fib6_entry_nexthop_add:
3934         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3935         return err;
3936 }
3937
3938 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
3939                                      struct rt6_info *rt)
3940 {
3941         struct mlxsw_sp_fib6_entry *fib6_entry;
3942         struct mlxsw_sp_fib_node *fib_node;
3943
3944         if (mlxsw_sp->router->aborted)
3945                 return;
3946
3947         if (mlxsw_sp_fib6_rt_should_ignore(rt))
3948                 return;
3949
3950         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
3951         if (WARN_ON(!fib6_entry))
3952                 return;
3953
3954         /* If route is part of a multipath entry, but not the last one
3955          * removed, then only reduce its nexthop group.
3956          */
3957         if (!list_is_singular(&fib6_entry->rt6_list)) {
3958                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
3959                 return;
3960         }
3961
3962         fib_node = fib6_entry->common.fib_node;
3963
3964         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
3965         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
3966         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3967 }
3968
3969 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
3970                                             enum mlxsw_reg_ralxx_protocol proto,
3971                                             u8 tree_id)
3972 {
3973         char ralta_pl[MLXSW_REG_RALTA_LEN];
3974         char ralst_pl[MLXSW_REG_RALST_LEN];
3975         int i, err;
3976
3977         mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
3978         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
3979         if (err)
3980                 return err;
3981
3982         mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
3983         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
3984         if (err)
3985                 return err;
3986
3987         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
3988                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
3989                 char raltb_pl[MLXSW_REG_RALTB_LEN];
3990                 char ralue_pl[MLXSW_REG_RALUE_LEN];
3991
3992                 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
3993                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
3994                                       raltb_pl);
3995                 if (err)
3996                         return err;
3997
3998                 mlxsw_reg_ralue_pack(ralue_pl, proto,
3999                                      MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
4000                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4001                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4002                                       ralue_pl);
4003                 if (err)
4004                         return err;
4005         }
4006
4007         return 0;
4008 }
4009
4010 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
4011 {
4012         enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
4013         int err;
4014
4015         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4016                                                MLXSW_SP_LPM_TREE_MIN);
4017         if (err)
4018                 return err;
4019
4020         proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
4021         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4022                                                 MLXSW_SP_LPM_TREE_MIN + 1);
4023 }
4024
4025 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
4026                                      struct mlxsw_sp_fib_node *fib_node)
4027 {
4028         struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
4029
4030         list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
4031                                  common.list) {
4032                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4033
4034                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4035                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4036                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4037                 /* Break when entry list is empty and node was freed.
4038                  * Otherwise, we'll access freed memory in the next
4039                  * iteration.
4040                  */
4041                 if (do_break)
4042                         break;
4043         }
4044 }
4045
4046 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
4047                                      struct mlxsw_sp_fib_node *fib_node)
4048 {
4049         struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
4050
4051         list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
4052                                  common.list) {
4053                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4054
4055                 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4056                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4057                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4058                 if (do_break)
4059                         break;
4060         }
4061 }
4062
4063 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
4064                                     struct mlxsw_sp_fib_node *fib_node)
4065 {
4066         switch (fib_node->fib->proto) {
4067         case MLXSW_SP_L3_PROTO_IPV4:
4068                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
4069                 break;
4070         case MLXSW_SP_L3_PROTO_IPV6:
4071                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
4072                 break;
4073         }
4074 }
4075
4076 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
4077                                   struct mlxsw_sp_vr *vr,
4078                                   enum mlxsw_sp_l3proto proto)
4079 {
4080         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
4081         struct mlxsw_sp_fib_node *fib_node, *tmp;
4082
4083         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
4084                 bool do_break = &tmp->list == &fib->node_list;
4085
4086                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
4087                 if (do_break)
4088                         break;
4089         }
4090 }
4091
4092 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
4093 {
4094         int i;
4095
4096         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4097                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4098
4099                 if (!mlxsw_sp_vr_is_used(vr))
4100                         continue;
4101                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
4102
4103                 /* If virtual router was only used for IPv4, then it's no
4104                  * longer used.
4105                  */
4106                 if (!mlxsw_sp_vr_is_used(vr))
4107                         continue;
4108                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
4109         }
4110 }
4111
4112 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
4113 {
4114         int err;
4115
4116         if (mlxsw_sp->router->aborted)
4117                 return;
4118         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
4119         mlxsw_sp_router_fib_flush(mlxsw_sp);
4120         mlxsw_sp->router->aborted = true;
4121         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
4122         if (err)
4123                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
4124 }
4125
4126 struct mlxsw_sp_fib_event_work {
4127         struct work_struct work;
4128         union {
4129                 struct fib6_entry_notifier_info fen6_info;
4130                 struct fib_entry_notifier_info fen_info;
4131                 struct fib_rule_notifier_info fr_info;
4132                 struct fib_nh_notifier_info fnh_info;
4133         };
4134         struct mlxsw_sp *mlxsw_sp;
4135         unsigned long event;
4136 };
4137
4138 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
4139 {
4140         struct mlxsw_sp_fib_event_work *fib_work =
4141                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4142         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4143         struct fib_rule *rule;
4144         bool replace, append;
4145         int err;
4146
4147         /* Protect internal structures from changes */
4148         rtnl_lock();
4149         switch (fib_work->event) {
4150         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4151         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4152         case FIB_EVENT_ENTRY_ADD:
4153                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4154                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
4155                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
4156                                                replace, append);
4157                 if (err)
4158                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4159                 fib_info_put(fib_work->fen_info.fi);
4160                 break;
4161         case FIB_EVENT_ENTRY_DEL:
4162                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
4163                 fib_info_put(fib_work->fen_info.fi);
4164                 break;
4165         case FIB_EVENT_RULE_ADD: /* fall through */
4166         case FIB_EVENT_RULE_DEL:
4167                 rule = fib_work->fr_info.rule;
4168                 if (!fib4_rule_default(rule) && !rule->l3mdev)
4169                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4170                 fib_rule_put(rule);
4171                 break;
4172         case FIB_EVENT_NH_ADD: /* fall through */
4173         case FIB_EVENT_NH_DEL:
4174                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
4175                                         fib_work->fnh_info.fib_nh);
4176                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
4177                 break;
4178         }
4179         rtnl_unlock();
4180         kfree(fib_work);
4181 }
4182
4183 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
4184 {
4185         struct mlxsw_sp_fib_event_work *fib_work =
4186                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4187         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4188         struct fib_rule *rule;
4189         bool replace;
4190         int err;
4191
4192         rtnl_lock();
4193         switch (fib_work->event) {
4194         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4195         case FIB_EVENT_ENTRY_ADD:
4196                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4197                 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
4198                                                fib_work->fen6_info.rt, replace);
4199                 if (err)
4200                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4201                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4202                 break;
4203         case FIB_EVENT_ENTRY_DEL:
4204                 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
4205                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4206                 break;
4207         case FIB_EVENT_RULE_ADD: /* fall through */
4208         case FIB_EVENT_RULE_DEL:
4209                 rule = fib_work->fr_info.rule;
4210                 if (!fib6_rule_default(rule) && !rule->l3mdev)
4211                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4212                 fib_rule_put(rule);
4213                 break;
4214         }
4215         rtnl_unlock();
4216         kfree(fib_work);
4217 }
4218
4219 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
4220                                        struct fib_notifier_info *info)
4221 {
4222         switch (fib_work->event) {
4223         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4224         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4225         case FIB_EVENT_ENTRY_ADD: /* fall through */
4226         case FIB_EVENT_ENTRY_DEL:
4227                 memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
4228                 /* Take referece on fib_info to prevent it from being
4229                  * freed while work is queued. Release it afterwards.
4230                  */
4231                 fib_info_hold(fib_work->fen_info.fi);
4232                 break;
4233         case FIB_EVENT_RULE_ADD: /* fall through */
4234         case FIB_EVENT_RULE_DEL:
4235                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4236                 fib_rule_get(fib_work->fr_info.rule);
4237                 break;
4238         case FIB_EVENT_NH_ADD: /* fall through */
4239         case FIB_EVENT_NH_DEL:
4240                 memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
4241                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
4242                 break;
4243         }
4244 }
4245
4246 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
4247                                        struct fib_notifier_info *info)
4248 {
4249         switch (fib_work->event) {
4250         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4251         case FIB_EVENT_ENTRY_ADD: /* fall through */
4252         case FIB_EVENT_ENTRY_DEL:
4253                 memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
4254                 rt6_hold(fib_work->fen6_info.rt);
4255                 break;
4256         case FIB_EVENT_RULE_ADD: /* fall through */
4257         case FIB_EVENT_RULE_DEL:
4258                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4259                 fib_rule_get(fib_work->fr_info.rule);
4260                 break;
4261         }
4262 }
4263
4264 /* Called with rcu_read_lock() */
4265 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
4266                                      unsigned long event, void *ptr)
4267 {
4268         struct mlxsw_sp_fib_event_work *fib_work;
4269         struct fib_notifier_info *info = ptr;
4270         struct mlxsw_sp_router *router;
4271
4272         if (!net_eq(info->net, &init_net))
4273                 return NOTIFY_DONE;
4274
4275         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
4276         if (WARN_ON(!fib_work))
4277                 return NOTIFY_BAD;
4278
4279         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
4280         fib_work->mlxsw_sp = router->mlxsw_sp;
4281         fib_work->event = event;
4282
4283         switch (info->family) {
4284         case AF_INET:
4285                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
4286                 mlxsw_sp_router_fib4_event(fib_work, info);
4287                 break;
4288         case AF_INET6:
4289                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
4290                 mlxsw_sp_router_fib6_event(fib_work, info);
4291                 break;
4292         }
4293
4294         mlxsw_core_schedule_work(&fib_work->work);
4295
4296         return NOTIFY_DONE;
4297 }
4298
4299 static struct mlxsw_sp_rif *
4300 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
4301                          const struct net_device *dev)
4302 {
4303         int i;
4304
4305         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
4306                 if (mlxsw_sp->router->rifs[i] &&
4307                     mlxsw_sp->router->rifs[i]->dev == dev)
4308                         return mlxsw_sp->router->rifs[i];
4309
4310         return NULL;
4311 }
4312
4313 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
4314 {
4315         char ritr_pl[MLXSW_REG_RITR_LEN];
4316         int err;
4317
4318         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
4319         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4320         if (WARN_ON_ONCE(err))
4321                 return err;
4322
4323         mlxsw_reg_ritr_enable_set(ritr_pl, false);
4324         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4325 }
4326
4327 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4328                                           struct mlxsw_sp_rif *rif)
4329 {
4330         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
4331         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
4332         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
4333 }
4334
4335 static bool
4336 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
4337                            unsigned long event)
4338 {
4339         struct inet6_dev *inet6_dev;
4340         bool addr_list_empty = true;
4341         struct in_device *idev;
4342
4343         switch (event) {
4344         case NETDEV_UP:
4345                 return rif == NULL;
4346         case NETDEV_DOWN:
4347                 idev = __in_dev_get_rtnl(dev);
4348                 if (idev && idev->ifa_list)
4349                         addr_list_empty = false;
4350
4351                 inet6_dev = __in6_dev_get(dev);
4352                 if (addr_list_empty && inet6_dev &&
4353                     !list_empty(&inet6_dev->addr_list))
4354                         addr_list_empty = false;
4355
4356                 if (rif && addr_list_empty &&
4357                     !netif_is_l3_slave(rif->dev))
4358                         return true;
4359                 /* It is possible we already removed the RIF ourselves
4360                  * if it was assigned to a netdev that is now a bridge
4361                  * or LAG slave.
4362                  */
4363                 return false;
4364         }
4365
4366         return false;
4367 }
4368
4369 static enum mlxsw_sp_rif_type
4370 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
4371                       const struct net_device *dev)
4372 {
4373         enum mlxsw_sp_fid_type type;
4374
4375         /* RIF type is derived from the type of the underlying FID */
4376         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
4377                 type = MLXSW_SP_FID_TYPE_8021Q;
4378         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
4379                 type = MLXSW_SP_FID_TYPE_8021Q;
4380         else if (netif_is_bridge_master(dev))
4381                 type = MLXSW_SP_FID_TYPE_8021D;
4382         else
4383                 type = MLXSW_SP_FID_TYPE_RFID;
4384
4385         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
4386 }
4387
4388 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
4389 {
4390         int i;
4391
4392         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
4393                 if (!mlxsw_sp->router->rifs[i]) {
4394                         *p_rif_index = i;
4395                         return 0;
4396                 }
4397         }
4398
4399         return -ENOBUFS;
4400 }
4401
4402 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
4403                                                u16 vr_id,
4404                                                struct net_device *l3_dev)
4405 {
4406         struct mlxsw_sp_rif *rif;
4407
4408         rif = kzalloc(rif_size, GFP_KERNEL);
4409         if (!rif)
4410                 return NULL;
4411
4412         INIT_LIST_HEAD(&rif->nexthop_list);
4413         INIT_LIST_HEAD(&rif->neigh_list);
4414         ether_addr_copy(rif->addr, l3_dev->dev_addr);
4415         rif->mtu = l3_dev->mtu;
4416         rif->vr_id = vr_id;
4417         rif->dev = l3_dev;
4418         rif->rif_index = rif_index;
4419
4420         return rif;
4421 }
4422
4423 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
4424                                            u16 rif_index)
4425 {
4426         return mlxsw_sp->router->rifs[rif_index];
4427 }
4428
4429 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
4430 {
4431         return rif->rif_index;
4432 }
4433
4434 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
4435 {
4436         return rif->dev->ifindex;
4437 }
4438
4439 static struct mlxsw_sp_rif *
4440 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
4441                     const struct mlxsw_sp_rif_params *params)
4442 {
4443         u32 tb_id = l3mdev_fib_table(params->dev);
4444         const struct mlxsw_sp_rif_ops *ops;
4445         enum mlxsw_sp_rif_type type;
4446         struct mlxsw_sp_rif *rif;
4447         struct mlxsw_sp_fid *fid;
4448         struct mlxsw_sp_vr *vr;
4449         u16 rif_index;
4450         int err;
4451
4452         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
4453         ops = mlxsw_sp->router->rif_ops_arr[type];
4454
4455         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
4456         if (IS_ERR(vr))
4457                 return ERR_CAST(vr);
4458
4459         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
4460         if (err)
4461                 goto err_rif_index_alloc;
4462
4463         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
4464         if (!rif) {
4465                 err = -ENOMEM;
4466                 goto err_rif_alloc;
4467         }
4468         rif->mlxsw_sp = mlxsw_sp;
4469         rif->ops = ops;
4470
4471         fid = ops->fid_get(rif);
4472         if (IS_ERR(fid)) {
4473                 err = PTR_ERR(fid);
4474                 goto err_fid_get;
4475         }
4476         rif->fid = fid;
4477
4478         if (ops->setup)
4479                 ops->setup(rif, params);
4480
4481         err = ops->configure(rif);
4482         if (err)
4483                 goto err_configure;
4484
4485         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, params->dev->dev_addr,
4486                                   mlxsw_sp_fid_index(fid), true);
4487         if (err)
4488                 goto err_rif_fdb_op;
4489
4490         mlxsw_sp_rif_counters_alloc(rif);
4491         mlxsw_sp_fid_rif_set(fid, rif);
4492         mlxsw_sp->router->rifs[rif_index] = rif;
4493         vr->rif_count++;
4494
4495         return rif;
4496
4497 err_rif_fdb_op:
4498         ops->deconfigure(rif);
4499 err_configure:
4500         mlxsw_sp_fid_put(fid);
4501 err_fid_get:
4502         kfree(rif);
4503 err_rif_alloc:
4504 err_rif_index_alloc:
4505         mlxsw_sp_vr_put(vr);
4506         return ERR_PTR(err);
4507 }
4508
4509 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
4510 {
4511         const struct mlxsw_sp_rif_ops *ops = rif->ops;
4512         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4513         struct mlxsw_sp_fid *fid = rif->fid;
4514         struct mlxsw_sp_vr *vr;
4515
4516         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
4517         vr = &mlxsw_sp->router->vrs[rif->vr_id];
4518
4519         vr->rif_count--;
4520         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
4521         mlxsw_sp_fid_rif_set(fid, NULL);
4522         mlxsw_sp_rif_counters_free(rif);
4523         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->dev->dev_addr,
4524                             mlxsw_sp_fid_index(fid), false);
4525         ops->deconfigure(rif);
4526         mlxsw_sp_fid_put(fid);
4527         kfree(rif);
4528         mlxsw_sp_vr_put(vr);
4529 }
4530
4531 static void
4532 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
4533                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4534 {
4535         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4536
4537         params->vid = mlxsw_sp_port_vlan->vid;
4538         params->lag = mlxsw_sp_port->lagged;
4539         if (params->lag)
4540                 params->lag_id = mlxsw_sp_port->lag_id;
4541         else
4542                 params->system_port = mlxsw_sp_port->local_port;
4543 }
4544
4545 static int
4546 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
4547                                struct net_device *l3_dev)
4548 {
4549         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4550         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
4551         u16 vid = mlxsw_sp_port_vlan->vid;
4552         struct mlxsw_sp_rif *rif;
4553         struct mlxsw_sp_fid *fid;
4554         int err;
4555
4556         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4557         if (!rif) {
4558                 struct mlxsw_sp_rif_params params = {
4559                         .dev = l3_dev,
4560                 };
4561
4562                 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
4563                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
4564                 if (IS_ERR(rif))
4565                         return PTR_ERR(rif);
4566         }
4567
4568         /* FID was already created, just take a reference */
4569         fid = rif->ops->fid_get(rif);
4570         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
4571         if (err)
4572                 goto err_fid_port_vid_map;
4573
4574         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
4575         if (err)
4576                 goto err_port_vid_learning_set;
4577
4578         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
4579                                         BR_STATE_FORWARDING);
4580         if (err)
4581                 goto err_port_vid_stp_set;
4582
4583         mlxsw_sp_port_vlan->fid = fid;
4584
4585         return 0;
4586
4587 err_port_vid_stp_set:
4588         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
4589 err_port_vid_learning_set:
4590         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
4591 err_fid_port_vid_map:
4592         mlxsw_sp_fid_put(fid);
4593         return err;
4594 }
4595
4596 void
4597 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4598 {
4599         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4600         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
4601         u16 vid = mlxsw_sp_port_vlan->vid;
4602
4603         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
4604                 return;
4605
4606         mlxsw_sp_port_vlan->fid = NULL;
4607         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
4608         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
4609         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
4610         /* If router port holds the last reference on the rFID, then the
4611          * associated Sub-port RIF will be destroyed.
4612          */
4613         mlxsw_sp_fid_put(fid);
4614 }
4615
4616 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
4617                                              struct net_device *port_dev,
4618                                              unsigned long event, u16 vid)
4619 {
4620         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
4621         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
4622
4623         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
4624         if (WARN_ON(!mlxsw_sp_port_vlan))
4625                 return -EINVAL;
4626
4627         switch (event) {
4628         case NETDEV_UP:
4629                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
4630                                                       l3_dev);
4631         case NETDEV_DOWN:
4632                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
4633                 break;
4634         }
4635
4636         return 0;
4637 }
4638
4639 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
4640                                         unsigned long event)
4641 {
4642         if (netif_is_bridge_port(port_dev) ||
4643             netif_is_lag_port(port_dev) ||
4644             netif_is_ovs_port(port_dev))
4645                 return 0;
4646
4647         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
4648 }
4649
4650 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
4651                                          struct net_device *lag_dev,
4652                                          unsigned long event, u16 vid)
4653 {
4654         struct net_device *port_dev;
4655         struct list_head *iter;
4656         int err;
4657
4658         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
4659                 if (mlxsw_sp_port_dev_check(port_dev)) {
4660                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
4661                                                                 port_dev,
4662                                                                 event, vid);
4663                         if (err)
4664                                 return err;
4665                 }
4666         }
4667
4668         return 0;
4669 }
4670
4671 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
4672                                        unsigned long event)
4673 {
4674         if (netif_is_bridge_port(lag_dev))
4675                 return 0;
4676
4677         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
4678 }
4679
4680 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
4681                                           unsigned long event)
4682 {
4683         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
4684         struct mlxsw_sp_rif_params params = {
4685                 .dev = l3_dev,
4686         };
4687         struct mlxsw_sp_rif *rif;
4688
4689         switch (event) {
4690         case NETDEV_UP:
4691                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
4692                 if (IS_ERR(rif))
4693                         return PTR_ERR(rif);
4694                 break;
4695         case NETDEV_DOWN:
4696                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4697                 mlxsw_sp_rif_destroy(rif);
4698                 break;
4699         }
4700
4701         return 0;
4702 }
4703
4704 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
4705                                         unsigned long event)
4706 {
4707         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
4708         u16 vid = vlan_dev_vlan_id(vlan_dev);
4709
4710         if (netif_is_bridge_port(vlan_dev))
4711                 return 0;
4712
4713         if (mlxsw_sp_port_dev_check(real_dev))
4714                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
4715                                                          event, vid);
4716         else if (netif_is_lag_master(real_dev))
4717                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
4718                                                      vid);
4719         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
4720                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
4721
4722         return 0;
4723 }
4724
4725 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
4726                                      unsigned long event)
4727 {
4728         if (mlxsw_sp_port_dev_check(dev))
4729                 return mlxsw_sp_inetaddr_port_event(dev, event);
4730         else if (netif_is_lag_master(dev))
4731                 return mlxsw_sp_inetaddr_lag_event(dev, event);
4732         else if (netif_is_bridge_master(dev))
4733                 return mlxsw_sp_inetaddr_bridge_event(dev, event);
4734         else if (is_vlan_dev(dev))
4735                 return mlxsw_sp_inetaddr_vlan_event(dev, event);
4736         else
4737                 return 0;
4738 }
4739
4740 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
4741                             unsigned long event, void *ptr)
4742 {
4743         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
4744         struct net_device *dev = ifa->ifa_dev->dev;
4745         struct mlxsw_sp *mlxsw_sp;
4746         struct mlxsw_sp_rif *rif;
4747         int err = 0;
4748
4749         mlxsw_sp = mlxsw_sp_lower_get(dev);
4750         if (!mlxsw_sp)
4751                 goto out;
4752
4753         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4754         if (!mlxsw_sp_rif_should_config(rif, dev, event))
4755                 goto out;
4756
4757         err = __mlxsw_sp_inetaddr_event(dev, event);
4758 out:
4759         return notifier_from_errno(err);
4760 }
4761
4762 struct mlxsw_sp_inet6addr_event_work {
4763         struct work_struct work;
4764         struct net_device *dev;
4765         unsigned long event;
4766 };
4767
4768 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
4769 {
4770         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
4771                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
4772         struct net_device *dev = inet6addr_work->dev;
4773         unsigned long event = inet6addr_work->event;
4774         struct mlxsw_sp *mlxsw_sp;
4775         struct mlxsw_sp_rif *rif;
4776
4777         rtnl_lock();
4778         mlxsw_sp = mlxsw_sp_lower_get(dev);
4779         if (!mlxsw_sp)
4780                 goto out;
4781
4782         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4783         if (!mlxsw_sp_rif_should_config(rif, dev, event))
4784                 goto out;
4785
4786         __mlxsw_sp_inetaddr_event(dev, event);
4787 out:
4788         rtnl_unlock();
4789         dev_put(dev);
4790         kfree(inet6addr_work);
4791 }
4792
4793 /* Called with rcu_read_lock() */
4794 int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
4795                              unsigned long event, void *ptr)
4796 {
4797         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
4798         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
4799         struct net_device *dev = if6->idev->dev;
4800
4801         if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
4802                 return NOTIFY_DONE;
4803
4804         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
4805         if (!inet6addr_work)
4806                 return NOTIFY_BAD;
4807
4808         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
4809         inet6addr_work->dev = dev;
4810         inet6addr_work->event = event;
4811         dev_hold(dev);
4812         mlxsw_core_schedule_work(&inet6addr_work->work);
4813
4814         return NOTIFY_DONE;
4815 }
4816
4817 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
4818                              const char *mac, int mtu)
4819 {
4820         char ritr_pl[MLXSW_REG_RITR_LEN];
4821         int err;
4822
4823         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
4824         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4825         if (err)
4826                 return err;
4827
4828         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
4829         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
4830         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
4831         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4832 }
4833
4834 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
4835 {
4836         struct mlxsw_sp *mlxsw_sp;
4837         struct mlxsw_sp_rif *rif;
4838         u16 fid_index;
4839         int err;
4840
4841         mlxsw_sp = mlxsw_sp_lower_get(dev);
4842         if (!mlxsw_sp)
4843                 return 0;
4844
4845         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4846         if (!rif)
4847                 return 0;
4848         fid_index = mlxsw_sp_fid_index(rif->fid);
4849
4850         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
4851         if (err)
4852                 return err;
4853
4854         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
4855                                 dev->mtu);
4856         if (err)
4857                 goto err_rif_edit;
4858
4859         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
4860         if (err)
4861                 goto err_rif_fdb_op;
4862
4863         ether_addr_copy(rif->addr, dev->dev_addr);
4864         rif->mtu = dev->mtu;
4865
4866         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
4867
4868         return 0;
4869
4870 err_rif_fdb_op:
4871         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
4872 err_rif_edit:
4873         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
4874         return err;
4875 }
4876
4877 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
4878                                   struct net_device *l3_dev)
4879 {
4880         struct mlxsw_sp_rif *rif;
4881
4882         /* If netdev is already associated with a RIF, then we need to
4883          * destroy it and create a new one with the new virtual router ID.
4884          */
4885         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4886         if (rif)
4887                 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
4888
4889         return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
4890 }
4891
4892 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
4893                                     struct net_device *l3_dev)
4894 {
4895         struct mlxsw_sp_rif *rif;
4896
4897         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4898         if (!rif)
4899                 return;
4900         __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
4901 }
4902
4903 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
4904                                  struct netdev_notifier_changeupper_info *info)
4905 {
4906         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
4907         int err = 0;
4908
4909         if (!mlxsw_sp)
4910                 return 0;
4911
4912         switch (event) {
4913         case NETDEV_PRECHANGEUPPER:
4914                 return 0;
4915         case NETDEV_CHANGEUPPER:
4916                 if (info->linking)
4917                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
4918                 else
4919                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
4920                 break;
4921         }
4922
4923         return err;
4924 }
4925
4926 static struct mlxsw_sp_rif_subport *
4927 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
4928 {
4929         return container_of(rif, struct mlxsw_sp_rif_subport, common);
4930 }
4931
4932 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
4933                                        const struct mlxsw_sp_rif_params *params)
4934 {
4935         struct mlxsw_sp_rif_subport *rif_subport;
4936
4937         rif_subport = mlxsw_sp_rif_subport_rif(rif);
4938         rif_subport->vid = params->vid;
4939         rif_subport->lag = params->lag;
4940         if (params->lag)
4941                 rif_subport->lag_id = params->lag_id;
4942         else
4943                 rif_subport->system_port = params->system_port;
4944 }
4945
4946 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
4947 {
4948         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4949         struct mlxsw_sp_rif_subport *rif_subport;
4950         char ritr_pl[MLXSW_REG_RITR_LEN];
4951
4952         rif_subport = mlxsw_sp_rif_subport_rif(rif);
4953         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
4954                             rif->rif_index, rif->vr_id, rif->dev->mtu);
4955         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
4956         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
4957                                   rif_subport->lag ? rif_subport->lag_id :
4958                                                      rif_subport->system_port,
4959                                   rif_subport->vid);
4960
4961         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4962 }
4963
4964 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
4965 {
4966         return mlxsw_sp_rif_subport_op(rif, true);
4967 }
4968
4969 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
4970 {
4971         mlxsw_sp_rif_subport_op(rif, false);
4972 }
4973
4974 static struct mlxsw_sp_fid *
4975 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
4976 {
4977         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
4978 }
4979
4980 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
4981         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
4982         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
4983         .setup                  = mlxsw_sp_rif_subport_setup,
4984         .configure              = mlxsw_sp_rif_subport_configure,
4985         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
4986         .fid_get                = mlxsw_sp_rif_subport_fid_get,
4987 };
4988
4989 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
4990                                     enum mlxsw_reg_ritr_if_type type,
4991                                     u16 vid_fid, bool enable)
4992 {
4993         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4994         char ritr_pl[MLXSW_REG_RITR_LEN];
4995
4996         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
4997                             rif->dev->mtu);
4998         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
4999         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
5000
5001         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5002 }
5003
5004 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
5005 {
5006         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
5007 }
5008
5009 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
5010 {
5011         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5012         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5013         int err;
5014
5015         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
5016         if (err)
5017                 return err;
5018
5019         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5020                                      mlxsw_sp_router_port(mlxsw_sp), true);
5021         if (err)
5022                 goto err_fid_mc_flood_set;
5023
5024         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5025                                      mlxsw_sp_router_port(mlxsw_sp), true);
5026         if (err)
5027                 goto err_fid_bc_flood_set;
5028
5029         return 0;
5030
5031 err_fid_bc_flood_set:
5032         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5033                                mlxsw_sp_router_port(mlxsw_sp), false);
5034 err_fid_mc_flood_set:
5035         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5036         return err;
5037 }
5038
5039 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
5040 {
5041         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5042         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5043
5044         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5045                                mlxsw_sp_router_port(mlxsw_sp), false);
5046         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5047                                mlxsw_sp_router_port(mlxsw_sp), false);
5048         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5049 }
5050
5051 static struct mlxsw_sp_fid *
5052 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
5053 {
5054         u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
5055
5056         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
5057 }
5058
5059 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
5060         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
5061         .rif_size               = sizeof(struct mlxsw_sp_rif),
5062         .configure              = mlxsw_sp_rif_vlan_configure,
5063         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
5064         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
5065 };
5066
5067 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
5068 {
5069         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5070         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5071         int err;
5072
5073         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
5074                                        true);
5075         if (err)
5076                 return err;
5077
5078         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5079                                      mlxsw_sp_router_port(mlxsw_sp), true);
5080         if (err)
5081                 goto err_fid_mc_flood_set;
5082
5083         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5084                                      mlxsw_sp_router_port(mlxsw_sp), true);
5085         if (err)
5086                 goto err_fid_bc_flood_set;
5087
5088         return 0;
5089
5090 err_fid_bc_flood_set:
5091         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5092                                mlxsw_sp_router_port(mlxsw_sp), false);
5093 err_fid_mc_flood_set:
5094         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5095         return err;
5096 }
5097
5098 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
5099 {
5100         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5101         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5102
5103         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5104                                mlxsw_sp_router_port(mlxsw_sp), false);
5105         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5106                                mlxsw_sp_router_port(mlxsw_sp), false);
5107         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5108 }
5109
5110 static struct mlxsw_sp_fid *
5111 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
5112 {
5113         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
5114 }
5115
5116 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
5117         .type                   = MLXSW_SP_RIF_TYPE_FID,
5118         .rif_size               = sizeof(struct mlxsw_sp_rif),
5119         .configure              = mlxsw_sp_rif_fid_configure,
5120         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
5121         .fid_get                = mlxsw_sp_rif_fid_fid_get,
5122 };
5123
5124 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
5125         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
5126         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_ops,
5127         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
5128 };
5129
5130 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
5131 {
5132         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5133
5134         mlxsw_sp->router->rifs = kcalloc(max_rifs,
5135                                          sizeof(struct mlxsw_sp_rif *),
5136                                          GFP_KERNEL);
5137         if (!mlxsw_sp->router->rifs)
5138                 return -ENOMEM;
5139
5140         mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
5141
5142         return 0;
5143 }
5144
5145 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
5146 {
5147         int i;
5148
5149         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5150                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
5151
5152         kfree(mlxsw_sp->router->rifs);
5153 }
5154
5155 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
5156 {
5157         struct mlxsw_sp_router *router;
5158
5159         /* Flush pending FIB notifications and then flush the device's
5160          * table before requesting another dump. The FIB notification
5161          * block is unregistered, so no need to take RTNL.
5162          */
5163         mlxsw_core_flush_owq();
5164         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
5165         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
5166 }
5167
5168 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5169 {
5170         char rgcr_pl[MLXSW_REG_RGCR_LEN];
5171         u64 max_rifs;
5172         int err;
5173
5174         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
5175                 return -EIO;
5176         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5177
5178         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
5179         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
5180         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5181         if (err)
5182                 return err;
5183         return 0;
5184 }
5185
5186 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5187 {
5188         char rgcr_pl[MLXSW_REG_RGCR_LEN];
5189
5190         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
5191         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5192 }
5193
5194 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5195 {
5196         struct mlxsw_sp_router *router;
5197         int err;
5198
5199         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
5200         if (!router)
5201                 return -ENOMEM;
5202         mlxsw_sp->router = router;
5203         router->mlxsw_sp = mlxsw_sp;
5204
5205         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
5206         err = __mlxsw_sp_router_init(mlxsw_sp);
5207         if (err)
5208                 goto err_router_init;
5209
5210         err = mlxsw_sp_rifs_init(mlxsw_sp);
5211         if (err)
5212                 goto err_rifs_init;
5213
5214         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
5215                               &mlxsw_sp_nexthop_ht_params);
5216         if (err)
5217                 goto err_nexthop_ht_init;
5218
5219         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
5220                               &mlxsw_sp_nexthop_group_ht_params);
5221         if (err)
5222                 goto err_nexthop_group_ht_init;
5223
5224         err = mlxsw_sp_lpm_init(mlxsw_sp);
5225         if (err)
5226                 goto err_lpm_init;
5227
5228         err = mlxsw_sp_vrs_init(mlxsw_sp);
5229         if (err)
5230                 goto err_vrs_init;
5231
5232         err = mlxsw_sp_neigh_init(mlxsw_sp);
5233         if (err)
5234                 goto err_neigh_init;
5235
5236         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
5237         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
5238                                     mlxsw_sp_router_fib_dump_flush);
5239         if (err)
5240                 goto err_register_fib_notifier;
5241
5242         return 0;
5243
5244 err_register_fib_notifier:
5245         mlxsw_sp_neigh_fini(mlxsw_sp);
5246 err_neigh_init:
5247         mlxsw_sp_vrs_fini(mlxsw_sp);
5248 err_vrs_init:
5249         mlxsw_sp_lpm_fini(mlxsw_sp);
5250 err_lpm_init:
5251         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
5252 err_nexthop_group_ht_init:
5253         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
5254 err_nexthop_ht_init:
5255         mlxsw_sp_rifs_fini(mlxsw_sp);
5256 err_rifs_init:
5257         __mlxsw_sp_router_fini(mlxsw_sp);
5258 err_router_init:
5259         kfree(mlxsw_sp->router);
5260         return err;
5261 }
5262
5263 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5264 {
5265         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
5266         mlxsw_sp_neigh_fini(mlxsw_sp);
5267         mlxsw_sp_vrs_fini(mlxsw_sp);
5268         mlxsw_sp_lpm_fini(mlxsw_sp);
5269         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
5270         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
5271         mlxsw_sp_rifs_fini(mlxsw_sp);
5272         __mlxsw_sp_router_fini(mlxsw_sp);
5273         kfree(mlxsw_sp->router);
5274 }