drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_atcam.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/err.h>
   6 #include <linux/errno.h>
   7 #include <linux/gfp.h>
   8 #include <linux/refcount.h>
   9 #include <linux/rhashtable.h>
  10
  11 #include "reg.h"
  12 #include "core.h"
  13 #include "spectrum.h"
  14 #include "spectrum_acl_tcam.h"
  15 #include "core_acl_flex_keys.h"
  16
  17 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START    0
  18 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END      5
  19
  20 struct mlxsw_sp_acl_atcam_lkey_id_ht_key {
  21         char enc_key[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; /* MSB blocks */
  22         u8 erp_id;
  23 };
  24
  25 struct mlxsw_sp_acl_atcam_lkey_id {
  26         struct rhash_head ht_node;
  27         struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key;
  28         refcount_t refcnt;
  29         u32 id;
  30 };
  31
  32 struct mlxsw_sp_acl_atcam_region_ops {
  33         int (*init)(struct mlxsw_sp_acl_atcam_region *aregion);
  34         void (*fini)(struct mlxsw_sp_acl_atcam_region *aregion);
  35         struct mlxsw_sp_acl_atcam_lkey_id *
  36                 (*lkey_id_get)(struct mlxsw_sp_acl_atcam_region *aregion,
  37                                char *enc_key, u8 erp_id);
  38         void (*lkey_id_put)(struct mlxsw_sp_acl_atcam_region *aregion,
  39                             struct mlxsw_sp_acl_atcam_lkey_id *lkey_id);
  40 };
  41
  42 struct mlxsw_sp_acl_atcam_region_generic {
  43         struct mlxsw_sp_acl_atcam_lkey_id dummy_lkey_id;
  44 };
  45
  46 struct mlxsw_sp_acl_atcam_region_12kb {
  47         struct rhashtable lkey_ht;
  48         unsigned int max_lkey_id;
  49         unsigned long *used_lkey_id;
  50 };
  51
  52 static const struct rhashtable_params mlxsw_sp_acl_atcam_lkey_id_ht_params = {
  53         .key_len = sizeof(struct mlxsw_sp_acl_atcam_lkey_id_ht_key),
  54         .key_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_key),
  55         .head_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_node),
  56 };
  57
  58 static const struct rhashtable_params mlxsw_sp_acl_atcam_entries_ht_params = {
  59         .key_len = sizeof(struct mlxsw_sp_acl_atcam_entry_ht_key),
  60         .key_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_key),
  61         .head_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_node),
  62 };
  63
  64 static bool
  65 mlxsw_sp_acl_atcam_is_centry(const struct mlxsw_sp_acl_atcam_entry *aentry)
  66 {
  67         return mlxsw_sp_acl_erp_mask_is_ctcam(aentry->erp_mask);
  68 }
  69
  70 static int
  71 mlxsw_sp_acl_atcam_region_generic_init(struct mlxsw_sp_acl_atcam_region *aregion)
  72 {
  73         struct mlxsw_sp_acl_atcam_region_generic *region_generic;
  74
  75         region_generic = kzalloc(sizeof(*region_generic), GFP_KERNEL);
  76         if (!region_generic)
  77                 return -ENOMEM;
  78
  79         refcount_set(&region_generic->dummy_lkey_id.refcnt, 1);
  80         aregion->priv = region_generic;
  81
  82         return 0;
  83 }
  84
  85 static void
  86 mlxsw_sp_acl_atcam_region_generic_fini(struct mlxsw_sp_acl_atcam_region *aregion)
  87 {
  88         kfree(aregion->priv);
  89 }
  90
  91 static struct mlxsw_sp_acl_atcam_lkey_id *
  92 mlxsw_sp_acl_atcam_generic_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
  93                                        char *enc_key, u8 erp_id)
  94 {
  95         struct mlxsw_sp_acl_atcam_region_generic *region_generic;
  96
  97         region_generic = aregion->priv;
  98         return &region_generic->dummy_lkey_id;
  99 }
 100
 101 static void
 102 mlxsw_sp_acl_atcam_generic_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
 103                                        struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
 104 {
 105 }
 106
 107 static const struct mlxsw_sp_acl_atcam_region_ops
 108 mlxsw_sp_acl_atcam_region_generic_ops = {
 109         .init           = mlxsw_sp_acl_atcam_region_generic_init,
 110         .fini           = mlxsw_sp_acl_atcam_region_generic_fini,
 111         .lkey_id_get    = mlxsw_sp_acl_atcam_generic_lkey_id_get,
 112         .lkey_id_put    = mlxsw_sp_acl_atcam_generic_lkey_id_put,
 113 };
 114
 115 static int
 116 mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion)
 117 {
 118         struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
 119         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb;
 120         size_t alloc_size;
 121         u64 max_lkey_id;
 122         int err;
 123
 124         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID))
 125                 return -EIO;
 126
 127         max_lkey_id = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID);
 128         region_12kb = kzalloc(sizeof(*region_12kb), GFP_KERNEL);
 129         if (!region_12kb)
 130                 return -ENOMEM;
 131
 132         alloc_size = BITS_TO_LONGS(max_lkey_id) * sizeof(unsigned long);
 133         region_12kb->used_lkey_id = kzalloc(alloc_size, GFP_KERNEL);
 134         if (!region_12kb->used_lkey_id) {
 135                 err = -ENOMEM;
 136                 goto err_used_lkey_id_alloc;
 137         }
 138
 139         err = rhashtable_init(&region_12kb->lkey_ht,
 140                               &mlxsw_sp_acl_atcam_lkey_id_ht_params);
 141         if (err)
 142                 goto err_rhashtable_init;
 143
 144         region_12kb->max_lkey_id = max_lkey_id;
 145         aregion->priv = region_12kb;
 146
 147         return 0;
 148
 149 err_rhashtable_init:
 150         kfree(region_12kb->used_lkey_id);
 151 err_used_lkey_id_alloc:
 152         kfree(region_12kb);
 153         return err;
 154 }
 155
 156 static void
 157 mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region *aregion)
 158 {
 159         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 160
 161         rhashtable_destroy(&region_12kb->lkey_ht);
 162         kfree(region_12kb->used_lkey_id);
 163         kfree(region_12kb);
 164 }
 165
 166 static struct mlxsw_sp_acl_atcam_lkey_id *
 167 mlxsw_sp_acl_atcam_lkey_id_create(struct mlxsw_sp_acl_atcam_region *aregion,
 168                                   struct mlxsw_sp_acl_atcam_lkey_id_ht_key *ht_key)
 169 {
 170         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 171         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
 172         u32 id;
 173         int err;
 174
 175         id = find_first_zero_bit(region_12kb->used_lkey_id,
 176                                  region_12kb->max_lkey_id);
 177         if (id < region_12kb->max_lkey_id)
 178                 __set_bit(id, region_12kb->used_lkey_id);
 179         else
 180                 return ERR_PTR(-ENOBUFS);
 181
 182         lkey_id = kzalloc(sizeof(*lkey_id), GFP_KERNEL);
 183         if (!lkey_id) {
 184                 err = -ENOMEM;
 185                 goto err_lkey_id_alloc;
 186         }
 187
 188         lkey_id->id = id;
 189         memcpy(&lkey_id->ht_key, ht_key, sizeof(*ht_key));
 190         refcount_set(&lkey_id->refcnt, 1);
 191
 192         err = rhashtable_insert_fast(&region_12kb->lkey_ht,
 193                                      &lkey_id->ht_node,
 194                                      mlxsw_sp_acl_atcam_lkey_id_ht_params);
 195         if (err)
 196                 goto err_rhashtable_insert;
 197
 198         return lkey_id;
 199
 200 err_rhashtable_insert:
 201         kfree(lkey_id);
 202 err_lkey_id_alloc:
 203         __clear_bit(id, region_12kb->used_lkey_id);
 204         return ERR_PTR(err);
 205 }
 206
 207 static void
 208 mlxsw_sp_acl_atcam_lkey_id_destroy(struct mlxsw_sp_acl_atcam_region *aregion,
 209                                    struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
 210 {
 211         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 212         u32 id = lkey_id->id;
 213
 214         rhashtable_remove_fast(&region_12kb->lkey_ht, &lkey_id->ht_node,
 215                                mlxsw_sp_acl_atcam_lkey_id_ht_params);
 216         kfree(lkey_id);
 217         __clear_bit(id, region_12kb->used_lkey_id);
 218 }
 219
 220 static struct mlxsw_sp_acl_atcam_lkey_id *
 221 mlxsw_sp_acl_atcam_12kb_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
 222                                     char *enc_key, u8 erp_id)
 223 {
 224         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 225         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 226         struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key = {{ 0 } };
 227         struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
 228         struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
 229         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
 230
 231         memcpy(ht_key.enc_key, enc_key, sizeof(ht_key.enc_key));
 232         mlxsw_afk_clear(afk, ht_key.enc_key,
 233                         MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START,
 234                         MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END);
 235         ht_key.erp_id = erp_id;
 236         lkey_id = rhashtable_lookup_fast(&region_12kb->lkey_ht, &ht_key,
 237                                          mlxsw_sp_acl_atcam_lkey_id_ht_params);
 238         if (lkey_id) {
 239                 refcount_inc(&lkey_id->refcnt);
 240                 return lkey_id;
 241         }
 242
 243         return mlxsw_sp_acl_atcam_lkey_id_create(aregion, &ht_key);
 244 }
 245
 246 static void
 247 mlxsw_sp_acl_atcam_12kb_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
 248                                     struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
 249 {
 250         if (refcount_dec_and_test(&lkey_id->refcnt))
 251                 mlxsw_sp_acl_atcam_lkey_id_destroy(aregion, lkey_id);
 252 }
 253
 254 static const struct mlxsw_sp_acl_atcam_region_ops
 255 mlxsw_sp_acl_atcam_region_12kb_ops = {
 256         .init           = mlxsw_sp_acl_atcam_region_12kb_init,
 257         .fini           = mlxsw_sp_acl_atcam_region_12kb_fini,
 258         .lkey_id_get    = mlxsw_sp_acl_atcam_12kb_lkey_id_get,
 259         .lkey_id_put    = mlxsw_sp_acl_atcam_12kb_lkey_id_put,
 260 };
 261
 262 static const struct mlxsw_sp_acl_atcam_region_ops *
 263 mlxsw_sp_acl_atcam_region_ops_arr[] = {
 264         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB]    =
 265                 &mlxsw_sp_acl_atcam_region_generic_ops,
 266         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB]    =
 267                 &mlxsw_sp_acl_atcam_region_generic_ops,
 268         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB]    =
 269                 &mlxsw_sp_acl_atcam_region_generic_ops,
 270         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB]   =
 271                 &mlxsw_sp_acl_atcam_region_12kb_ops,
 272 };
 273
 274 int mlxsw_sp_acl_atcam_region_associate(struct mlxsw_sp *mlxsw_sp,
 275                                         u16 region_id)
 276 {
 277         char perar_pl[MLXSW_REG_PERAR_LEN];
 278         /* For now, just assume that every region has 12 key blocks */
 279         u16 hw_region = region_id * 3;
 280         u64 max_regions;
 281
 282         max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);
 283         if (hw_region >= max_regions)
 284                 return -ENOBUFS;
 285
 286         mlxsw_reg_perar_pack(perar_pl, region_id, hw_region);
 287         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perar), perar_pl);
 288 }
 289
 290 static void
 291 mlxsw_sp_acl_atcam_region_type_init(struct mlxsw_sp_acl_atcam_region *aregion)
 292 {
 293         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 294         enum mlxsw_sp_acl_atcam_region_type region_type;
 295         unsigned int blocks_count;
 296
 297         /* We already know the blocks count can not exceed the maximum
 298          * blocks count.
 299          */
 300         blocks_count = mlxsw_afk_key_info_blocks_count_get(region->key_info);
 301         if (blocks_count <= 2)
 302                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB;
 303         else if (blocks_count <= 4)
 304                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB;
 305         else if (blocks_count <= 8)
 306                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB;
 307         else
 308                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB;
 309
 310         aregion->type = region_type;
 311         aregion->ops = mlxsw_sp_acl_atcam_region_ops_arr[region_type];
 312 }
 313
 314 int
 315 mlxsw_sp_acl_atcam_region_init(struct mlxsw_sp *mlxsw_sp,
 316                                struct mlxsw_sp_acl_atcam *atcam,
 317                                struct mlxsw_sp_acl_atcam_region *aregion,
 318                                struct mlxsw_sp_acl_tcam_region *region,
 319                                const struct mlxsw_sp_acl_ctcam_region_ops *ops)
 320 {
 321         int err;
 322
 323         aregion->region = region;
 324         aregion->atcam = atcam;
 325         mlxsw_sp_acl_atcam_region_type_init(aregion);
 326         INIT_LIST_HEAD(&aregion->entries_list);
 327
 328         err = rhashtable_init(&aregion->entries_ht,
 329                               &mlxsw_sp_acl_atcam_entries_ht_params);
 330         if (err)
 331                 return err;
 332         err = aregion->ops->init(aregion);
 333         if (err)
 334                 goto err_ops_init;
 335         err = mlxsw_sp_acl_erp_region_init(aregion);
 336         if (err)
 337                 goto err_erp_region_init;
 338         err = mlxsw_sp_acl_ctcam_region_init(mlxsw_sp, &aregion->cregion,
 339                                              region, ops);
 340         if (err)
 341                 goto err_ctcam_region_init;
 342
 343         return 0;
 344
 345 err_ctcam_region_init:
 346         mlxsw_sp_acl_erp_region_fini(aregion);
 347 err_erp_region_init:
 348         aregion->ops->fini(aregion);
 349 err_ops_init:
 350         rhashtable_destroy(&aregion->entries_ht);
 351         return err;
 352 }
 353
 354 void mlxsw_sp_acl_atcam_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
 355 {
 356         mlxsw_sp_acl_ctcam_region_fini(&aregion->cregion);
 357         mlxsw_sp_acl_erp_region_fini(aregion);
 358         aregion->ops->fini(aregion);
 359         rhashtable_destroy(&aregion->entries_ht);
 360         WARN_ON(!list_empty(&aregion->entries_list));
 361 }
 362
 363 void mlxsw_sp_acl_atcam_chunk_init(struct mlxsw_sp_acl_atcam_region *aregion,
 364                                    struct mlxsw_sp_acl_atcam_chunk *achunk,
 365                                    unsigned int priority)
 366 {
 367         mlxsw_sp_acl_ctcam_chunk_init(&aregion->cregion, &achunk->cchunk,
 368                                       priority);
 369 }
 370
 371 void mlxsw_sp_acl_atcam_chunk_fini(struct mlxsw_sp_acl_atcam_chunk *achunk)
 372 {
 373         mlxsw_sp_acl_ctcam_chunk_fini(&achunk->cchunk);
 374 }
 375
 376 static int
 377 mlxsw_sp_acl_atcam_region_entry_insert(struct mlxsw_sp *mlxsw_sp,
 378                                        struct mlxsw_sp_acl_atcam_region *aregion,
 379                                        struct mlxsw_sp_acl_atcam_entry *aentry,
 380                                        struct mlxsw_sp_acl_rule_info *rulei)
 381 {
 382         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 383         u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
 384         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
 385         char ptce3_pl[MLXSW_REG_PTCE3_LEN];
 386         u32 kvdl_index, priority;
 387         int err;
 388
 389         err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
 390         if (err)
 391                 return err;
 392
 393         lkey_id = aregion->ops->lkey_id_get(aregion, aentry->ht_key.enc_key,
 394                                             erp_id);
 395         if (IS_ERR(lkey_id))
 396                 return PTR_ERR(lkey_id);
 397         aentry->lkey_id = lkey_id;
 398
 399         kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
 400         mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_WRITE,
 401                              priority, region->tcam_region_info,
 402                              aentry->ht_key.enc_key, erp_id,
 403                              aentry->delta_info.start,
 404                              aentry->delta_info.mask,
 405                              aentry->delta_info.value,
 406                              refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
 407                              kvdl_index);
 408         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
 409         if (err)
 410                 goto err_ptce3_write;
 411
 412         return 0;
 413
 414 err_ptce3_write:
 415         aregion->ops->lkey_id_put(aregion, lkey_id);
 416         return err;
 417 }
 418
 419 static void
 420 mlxsw_sp_acl_atcam_region_entry_remove(struct mlxsw_sp *mlxsw_sp,
 421                                        struct mlxsw_sp_acl_atcam_region *aregion,
 422                                        struct mlxsw_sp_acl_atcam_entry *aentry)
 423 {
 424         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
 425         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 426         u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
 427         char *enc_key = aentry->ht_key.enc_key;
 428         char ptce3_pl[MLXSW_REG_PTCE3_LEN];
 429
 430         mlxsw_reg_ptce3_pack(ptce3_pl, false, MLXSW_REG_PTCE3_OP_WRITE_WRITE, 0,
 431                              region->tcam_region_info,
 432                              enc_key, erp_id,
 433                              aentry->delta_info.start,
 434                              aentry->delta_info.mask,
 435                              aentry->delta_info.value,
 436                              refcount_read(&lkey_id->refcnt) != 1,
 437                              lkey_id->id, 0);
 438         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
 439         aregion->ops->lkey_id_put(aregion, lkey_id);
 440 }
 441
 442 static int
 443 mlxsw_sp_acl_atcam_region_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
 444                                                struct mlxsw_sp_acl_atcam_region *aregion,
 445                                                struct mlxsw_sp_acl_atcam_entry *aentry,
 446                                                struct mlxsw_sp_acl_rule_info *rulei)
 447 {
 448         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
 449         u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
 450         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 451         char ptce3_pl[MLXSW_REG_PTCE3_LEN];
 452         u32 kvdl_index, priority;
 453         int err;
 454
 455         err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
 456         if (err)
 457                 return err;
 458         kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
 459         mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_UPDATE,
 460                              priority, region->tcam_region_info,
 461                              aentry->ht_key.enc_key, erp_id,
 462                              aentry->delta_info.start,
 463                              aentry->delta_info.mask,
 464                              aentry->delta_info.value,
 465                              refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
 466                              kvdl_index);
 467         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
 468 }
 469
 470 static int
 471 __mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
 472                                struct mlxsw_sp_acl_atcam_region *aregion,
 473                                struct mlxsw_sp_acl_atcam_entry *aentry,
 474                                struct mlxsw_sp_acl_rule_info *rulei)
 475 {
 476         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 477         char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
 478         struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
 479         const struct mlxsw_sp_acl_erp_delta *delta;
 480         struct mlxsw_sp_acl_erp_mask *erp_mask;
 481         int err;
 482
 483         mlxsw_afk_encode(afk, region->key_info, &rulei->values,
 484                          aentry->full_enc_key, mask);
 485
 486         erp_mask = mlxsw_sp_acl_erp_mask_get(aregion, mask, false);
 487         if (IS_ERR(erp_mask))
 488                 return PTR_ERR(erp_mask);
 489         aentry->erp_mask = erp_mask;
 490         aentry->ht_key.erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask);
 491         memcpy(aentry->ht_key.enc_key, aentry->full_enc_key,
 492                sizeof(aentry->ht_key.enc_key));
 493
 494         /* Compute all needed delta information and clear the delta bits
 495          * from the encrypted key.
 496          */
 497         delta = mlxsw_sp_acl_erp_delta(aentry->erp_mask);
 498         aentry->delta_info.start = mlxsw_sp_acl_erp_delta_start(delta);
 499         aentry->delta_info.mask = mlxsw_sp_acl_erp_delta_mask(delta);
 500         aentry->delta_info.value =
 501                 mlxsw_sp_acl_erp_delta_value(delta, aentry->full_enc_key);
 502         mlxsw_sp_acl_erp_delta_clear(delta, aentry->ht_key.enc_key);
 503
 504         /* Add rule to the list of A-TCAM rules, assuming this
 505          * rule is intended to A-TCAM. In case this rule does
 506          * not fit into A-TCAM it will be removed from the list.
 507          */
 508         list_add(&aentry->list, &aregion->entries_list);
 509
 510         /* We can't insert identical rules into the A-TCAM, so fail and
 511          * let the rule spill into C-TCAM
 512          */
 513         err = rhashtable_lookup_insert_fast(&aregion->entries_ht,
 514                                             &aentry->ht_node,
 515                                             mlxsw_sp_acl_atcam_entries_ht_params);
 516         if (err)
 517                 goto err_rhashtable_insert;
 518
 519         /* Bloom filter must be updated here, before inserting the rule into
 520          * the A-TCAM.
 521          */
 522         err = mlxsw_sp_acl_erp_bf_insert(mlxsw_sp, aregion, erp_mask, aentry);
 523         if (err)
 524                 goto err_bf_insert;
 525
 526         err = mlxsw_sp_acl_atcam_region_entry_insert(mlxsw_sp, aregion, aentry,
 527                                                      rulei);
 528         if (err)
 529                 goto err_rule_insert;
 530
 531         return 0;
 532
 533 err_rule_insert:
 534         mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, erp_mask, aentry);
 535 err_bf_insert:
 536         rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
 537                                mlxsw_sp_acl_atcam_entries_ht_params);
 538 err_rhashtable_insert:
 539         list_del(&aentry->list);
 540         mlxsw_sp_acl_erp_mask_put(aregion, erp_mask);
 541         return err;
 542 }
 543
 544 static void
 545 __mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
 546                                struct mlxsw_sp_acl_atcam_region *aregion,
 547                                struct mlxsw_sp_acl_atcam_entry *aentry)
 548 {
 549         mlxsw_sp_acl_atcam_region_entry_remove(mlxsw_sp, aregion, aentry);
 550         mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, aentry->erp_mask, aentry);
 551         rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
 552                                mlxsw_sp_acl_atcam_entries_ht_params);
 553         list_del(&aentry->list);
 554         mlxsw_sp_acl_erp_mask_put(aregion, aentry->erp_mask);
 555 }
 556
 557 static int
 558 __mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
 559                                           struct mlxsw_sp_acl_atcam_region *aregion,
 560                                           struct mlxsw_sp_acl_atcam_entry *aentry,
 561                                           struct mlxsw_sp_acl_rule_info *rulei)
 562 {
 563         return mlxsw_sp_acl_atcam_region_entry_action_replace(mlxsw_sp, aregion,
 564                                                               aentry, rulei);
 565 }
 566
 567 int mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
 568                                  struct mlxsw_sp_acl_atcam_region *aregion,
 569                                  struct mlxsw_sp_acl_atcam_chunk *achunk,
 570                                  struct mlxsw_sp_acl_atcam_entry *aentry,
 571                                  struct mlxsw_sp_acl_rule_info *rulei)
 572 {
 573         int err;
 574
 575         err = __mlxsw_sp_acl_atcam_entry_add(mlxsw_sp, aregion, aentry, rulei);
 576         if (!err)
 577                 return 0;
 578
 579         /* It is possible we failed to add the rule to the A-TCAM due to
 580          * exceeded number of masks. Try to spill into C-TCAM.
 581          */
 582         err = mlxsw_sp_acl_ctcam_entry_add(mlxsw_sp, &aregion->cregion,
 583                                            &achunk->cchunk, &aentry->centry,
 584                                            rulei, true);
 585         if (!err)
 586                 return 0;
 587
 588         return err;
 589 }
 590
 591 void mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
 592                                   struct mlxsw_sp_acl_atcam_region *aregion,
 593                                   struct mlxsw_sp_acl_atcam_chunk *achunk,
 594                                   struct mlxsw_sp_acl_atcam_entry *aentry)
 595 {
 596         if (mlxsw_sp_acl_atcam_is_centry(aentry))
 597                 mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, &aregion->cregion,
 598                                              &achunk->cchunk, &aentry->centry);
 599         else
 600                 __mlxsw_sp_acl_atcam_entry_del(mlxsw_sp, aregion, aentry);
 601 }
 602
 603 int
 604 mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
 605                                         struct mlxsw_sp_acl_atcam_region *aregion,
 606                                         struct mlxsw_sp_acl_atcam_chunk *achunk,
 607                                         struct mlxsw_sp_acl_atcam_entry *aentry,
 608                                         struct mlxsw_sp_acl_rule_info *rulei)
 609 {
 610         int err;
 611
 612         if (mlxsw_sp_acl_atcam_is_centry(aentry))
 613                 err = mlxsw_sp_acl_ctcam_entry_action_replace(mlxsw_sp,
 614                                                               &aregion->cregion,
 615                                                               &achunk->cchunk,
 616                                                               &aentry->centry,
 617                                                               rulei);
 618         else
 619                 err = __mlxsw_sp_acl_atcam_entry_action_replace(mlxsw_sp,
 620                                                                 aregion, aentry,
 621                                                                 rulei);
 622
 623         return err;
 624 }
 625
 626 int mlxsw_sp_acl_atcam_init(struct mlxsw_sp *mlxsw_sp,
 627                             struct mlxsw_sp_acl_atcam *atcam)
 628 {
 629         return mlxsw_sp_acl_erps_init(mlxsw_sp, atcam);
 630 }
 631
 632 void mlxsw_sp_acl_atcam_fini(struct mlxsw_sp *mlxsw_sp,
 633                              struct mlxsw_sp_acl_atcam *atcam)
 634 {
 635         mlxsw_sp_acl_erps_fini(mlxsw_sp, atcam);
 636 }