net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27
  28 unsigned int nf_tables_net_id __read_mostly;
  29
  30 static LIST_HEAD(nf_tables_expressions);
  31 static LIST_HEAD(nf_tables_objects);
  32 static LIST_HEAD(nf_tables_flowtables);
  33 static LIST_HEAD(nf_tables_destroy_list);
  34 static LIST_HEAD(nf_tables_gc_list);
  35 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  36 static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
  37
  38 enum {
  39         NFT_VALIDATE_SKIP       = 0,
  40         NFT_VALIDATE_NEED,
  41         NFT_VALIDATE_DO,
  42 };
  43
  44 static struct rhltable nft_objname_ht;
  45
  46 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  47 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  48 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  49
  50 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  51 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  52 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  53
  54 static const struct rhashtable_params nft_chain_ht_params = {
  55         .head_offset            = offsetof(struct nft_chain, rhlhead),
  56         .key_offset             = offsetof(struct nft_chain, name),
  57         .hashfn                 = nft_chain_hash,
  58         .obj_hashfn             = nft_chain_hash_obj,
  59         .obj_cmpfn              = nft_chain_hash_cmp,
  60         .automatic_shrinking    = true,
  61 };
  62
  63 static const struct rhashtable_params nft_objname_ht_params = {
  64         .head_offset            = offsetof(struct nft_object, rhlhead),
  65         .key_offset             = offsetof(struct nft_object, key),
  66         .hashfn                 = nft_objname_hash,
  67         .obj_hashfn             = nft_objname_hash_obj,
  68         .obj_cmpfn              = nft_objname_hash_cmp,
  69         .automatic_shrinking    = true,
  70 };
  71
  72 struct nft_audit_data {
  73         struct nft_table *table;
  74         int entries;
  75         int op;
  76         struct list_head list;
  77 };
  78
  79 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  80         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  81         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  82         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  83         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  84         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  85         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  86         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  87         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  88         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  89         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  90         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  91         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  92         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  93         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  94         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  95         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  96         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  97         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
  99         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
 100         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
 101         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 102         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 103         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 104         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 105         [NFT_MSG_GETSETELEM_RESET] = AUDIT_NFT_OP_SETELEM_RESET,
 106 };
 107
 108 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 109 {
 110         switch (table->validate_state) {
 111         case NFT_VALIDATE_SKIP:
 112                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 113                 break;
 114         case NFT_VALIDATE_NEED:
 115                 break;
 116         case NFT_VALIDATE_DO:
 117                 if (new_validate_state == NFT_VALIDATE_NEED)
 118                         return;
 119         }
 120
 121         table->validate_state = new_validate_state;
 122 }
 123 static void nf_tables_trans_destroy_work(struct work_struct *w);
 124 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 125
 126 static void nft_trans_gc_work(struct work_struct *work);
 127 static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
 128
 129 static void nft_ctx_init(struct nft_ctx *ctx,
 130                          struct net *net,
 131                          const struct sk_buff *skb,
 132                          const struct nlmsghdr *nlh,
 133                          u8 family,
 134                          struct nft_table *table,
 135                          struct nft_chain *chain,
 136                          const struct nlattr * const *nla)
 137 {
 138         ctx->net        = net;
 139         ctx->family     = family;
 140         ctx->level      = 0;
 141         ctx->table      = table;
 142         ctx->chain      = chain;
 143         ctx->nla        = nla;
 144         ctx->portid     = NETLINK_CB(skb).portid;
 145         ctx->report     = nlmsg_report(nlh);
 146         ctx->flags      = nlh->nlmsg_flags;
 147         ctx->seq        = nlh->nlmsg_seq;
 148 }
 149
 150 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 151                                              int msg_type, u32 size, gfp_t gfp)
 152 {
 153         struct nft_trans *trans;
 154
 155         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 156         if (trans == NULL)
 157                 return NULL;
 158
 159         INIT_LIST_HEAD(&trans->list);
 160         INIT_LIST_HEAD(&trans->binding_list);
 161         trans->msg_type = msg_type;
 162         trans->ctx      = *ctx;
 163
 164         return trans;
 165 }
 166
 167 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 168                                          int msg_type, u32 size)
 169 {
 170         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 171 }
 172
 173 static void nft_trans_list_del(struct nft_trans *trans)
 174 {
 175         list_del(&trans->list);
 176         list_del(&trans->binding_list);
 177 }
 178
 179 static void nft_trans_destroy(struct nft_trans *trans)
 180 {
 181         nft_trans_list_del(trans);
 182         kfree(trans);
 183 }
 184
 185 static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
 186                                  bool bind)
 187 {
 188         struct nftables_pernet *nft_net;
 189         struct net *net = ctx->net;
 190         struct nft_trans *trans;
 191
 192         if (!nft_set_is_anonymous(set))
 193                 return;
 194
 195         nft_net = nft_pernet(net);
 196         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 197                 switch (trans->msg_type) {
 198                 case NFT_MSG_NEWSET:
 199                         if (nft_trans_set(trans) == set)
 200                                 nft_trans_set_bound(trans) = bind;
 201                         break;
 202                 case NFT_MSG_NEWSETELEM:
 203                         if (nft_trans_elem_set(trans) == set)
 204                                 nft_trans_elem_set_bound(trans) = bind;
 205                         break;
 206                 }
 207         }
 208 }
 209
 210 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 211 {
 212         return __nft_set_trans_bind(ctx, set, true);
 213 }
 214
 215 static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
 216 {
 217         return __nft_set_trans_bind(ctx, set, false);
 218 }
 219
 220 static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
 221                                    struct nft_chain *chain, bool bind)
 222 {
 223         struct nftables_pernet *nft_net;
 224         struct net *net = ctx->net;
 225         struct nft_trans *trans;
 226
 227         if (!nft_chain_binding(chain))
 228                 return;
 229
 230         nft_net = nft_pernet(net);
 231         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 232                 switch (trans->msg_type) {
 233                 case NFT_MSG_NEWCHAIN:
 234                         if (nft_trans_chain(trans) == chain)
 235                                 nft_trans_chain_bound(trans) = bind;
 236                         break;
 237                 case NFT_MSG_NEWRULE:
 238                         if (trans->ctx.chain == chain)
 239                                 nft_trans_rule_bound(trans) = bind;
 240                         break;
 241                 }
 242         }
 243 }
 244
 245 static void nft_chain_trans_bind(const struct nft_ctx *ctx,
 246                                  struct nft_chain *chain)
 247 {
 248         __nft_chain_trans_bind(ctx, chain, true);
 249 }
 250
 251 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 252 {
 253         if (!nft_chain_binding(chain))
 254                 return 0;
 255
 256         if (nft_chain_binding(ctx->chain))
 257                 return -EOPNOTSUPP;
 258
 259         if (chain->bound)
 260                 return -EBUSY;
 261
 262         if (!nft_use_inc(&chain->use))
 263                 return -EMFILE;
 264
 265         chain->bound = true;
 266         nft_chain_trans_bind(ctx, chain);
 267
 268         return 0;
 269 }
 270
 271 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 272 {
 273         __nft_chain_trans_bind(ctx, chain, false);
 274 }
 275
 276 static int nft_netdev_register_hooks(struct net *net,
 277                                      struct list_head *hook_list)
 278 {
 279         struct nft_hook *hook;
 280         int err, j;
 281
 282         j = 0;
 283         list_for_each_entry(hook, hook_list, list) {
 284                 err = nf_register_net_hook(net, &hook->ops);
 285                 if (err < 0)
 286                         goto err_register;
 287
 288                 j++;
 289         }
 290         return 0;
 291
 292 err_register:
 293         list_for_each_entry(hook, hook_list, list) {
 294                 if (j-- <= 0)
 295                         break;
 296
 297                 nf_unregister_net_hook(net, &hook->ops);
 298         }
 299         return err;
 300 }
 301
 302 static void nft_netdev_unregister_hooks(struct net *net,
 303                                         struct list_head *hook_list,
 304                                         bool release_netdev)
 305 {
 306         struct nft_hook *hook, *next;
 307
 308         list_for_each_entry_safe(hook, next, hook_list, list) {
 309                 nf_unregister_net_hook(net, &hook->ops);
 310                 if (release_netdev) {
 311                         list_del(&hook->list);
 312                         kfree_rcu(hook, rcu);
 313                 }
 314         }
 315 }
 316
 317 static int nf_tables_register_hook(struct net *net,
 318                                    const struct nft_table *table,
 319                                    struct nft_chain *chain)
 320 {
 321         struct nft_base_chain *basechain;
 322         const struct nf_hook_ops *ops;
 323
 324         if (table->flags & NFT_TABLE_F_DORMANT ||
 325             !nft_is_base_chain(chain))
 326                 return 0;
 327
 328         basechain = nft_base_chain(chain);
 329         ops = &basechain->ops;
 330
 331         if (basechain->type->ops_register)
 332                 return basechain->type->ops_register(net, ops);
 333
 334         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 335                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 336
 337         return nf_register_net_hook(net, &basechain->ops);
 338 }
 339
 340 static void __nf_tables_unregister_hook(struct net *net,
 341                                         const struct nft_table *table,
 342                                         struct nft_chain *chain,
 343                                         bool release_netdev)
 344 {
 345         struct nft_base_chain *basechain;
 346         const struct nf_hook_ops *ops;
 347
 348         if (table->flags & NFT_TABLE_F_DORMANT ||
 349             !nft_is_base_chain(chain))
 350                 return;
 351         basechain = nft_base_chain(chain);
 352         ops = &basechain->ops;
 353
 354         if (basechain->type->ops_unregister)
 355                 return basechain->type->ops_unregister(net, ops);
 356
 357         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 358                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 359                                             release_netdev);
 360         else
 361                 nf_unregister_net_hook(net, &basechain->ops);
 362 }
 363
 364 static void nf_tables_unregister_hook(struct net *net,
 365                                       const struct nft_table *table,
 366                                       struct nft_chain *chain)
 367 {
 368         return __nf_tables_unregister_hook(net, table, chain, false);
 369 }
 370
 371 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 372 {
 373         struct nftables_pernet *nft_net = nft_pernet(net);
 374
 375         switch (trans->msg_type) {
 376         case NFT_MSG_NEWSET:
 377                 if (!nft_trans_set_update(trans) &&
 378                     nft_set_is_anonymous(nft_trans_set(trans)))
 379                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 380                 break;
 381         case NFT_MSG_NEWCHAIN:
 382                 if (!nft_trans_chain_update(trans) &&
 383                     nft_chain_binding(nft_trans_chain(trans)))
 384                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 385                 break;
 386         }
 387
 388         list_add_tail(&trans->list, &nft_net->commit_list);
 389 }
 390
 391 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 392 {
 393         struct nft_trans *trans;
 394
 395         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 396         if (trans == NULL)
 397                 return -ENOMEM;
 398
 399         if (msg_type == NFT_MSG_NEWTABLE)
 400                 nft_activate_next(ctx->net, ctx->table);
 401
 402         nft_trans_commit_list_add_tail(ctx->net, trans);
 403         return 0;
 404 }
 405
 406 static int nft_deltable(struct nft_ctx *ctx)
 407 {
 408         int err;
 409
 410         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 411         if (err < 0)
 412                 return err;
 413
 414         nft_deactivate_next(ctx->net, ctx->table);
 415         return err;
 416 }
 417
 418 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 419 {
 420         struct nft_trans *trans;
 421
 422         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 423         if (trans == NULL)
 424                 return ERR_PTR(-ENOMEM);
 425
 426         if (msg_type == NFT_MSG_NEWCHAIN) {
 427                 nft_activate_next(ctx->net, ctx->chain);
 428
 429                 if (ctx->nla[NFTA_CHAIN_ID]) {
 430                         nft_trans_chain_id(trans) =
 431                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 432                 }
 433         }
 434         nft_trans_chain(trans) = ctx->chain;
 435         nft_trans_commit_list_add_tail(ctx->net, trans);
 436
 437         return trans;
 438 }
 439
 440 static int nft_delchain(struct nft_ctx *ctx)
 441 {
 442         struct nft_trans *trans;
 443
 444         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 445         if (IS_ERR(trans))
 446                 return PTR_ERR(trans);
 447
 448         nft_use_dec(&ctx->table->use);
 449         nft_deactivate_next(ctx->net, ctx->chain);
 450
 451         return 0;
 452 }
 453
 454 void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
 455 {
 456         struct nft_expr *expr;
 457
 458         expr = nft_expr_first(rule);
 459         while (nft_expr_more(rule, expr)) {
 460                 if (expr->ops->activate)
 461                         expr->ops->activate(ctx, expr);
 462
 463                 expr = nft_expr_next(expr);
 464         }
 465 }
 466
 467 void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
 468                               enum nft_trans_phase phase)
 469 {
 470         struct nft_expr *expr;
 471
 472         expr = nft_expr_first(rule);
 473         while (nft_expr_more(rule, expr)) {
 474                 if (expr->ops->deactivate)
 475                         expr->ops->deactivate(ctx, expr, phase);
 476
 477                 expr = nft_expr_next(expr);
 478         }
 479 }
 480
 481 static int
 482 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 483 {
 484         /* You cannot delete the same rule twice */
 485         if (nft_is_active_next(ctx->net, rule)) {
 486                 nft_deactivate_next(ctx->net, rule);
 487                 nft_use_dec(&ctx->chain->use);
 488                 return 0;
 489         }
 490         return -ENOENT;
 491 }
 492
 493 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 494                                             struct nft_rule *rule)
 495 {
 496         struct nft_trans *trans;
 497
 498         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 499         if (trans == NULL)
 500                 return NULL;
 501
 502         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 503                 nft_trans_rule_id(trans) =
 504                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 505         }
 506         nft_trans_rule(trans) = rule;
 507         nft_trans_commit_list_add_tail(ctx->net, trans);
 508
 509         return trans;
 510 }
 511
 512 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 513 {
 514         struct nft_flow_rule *flow;
 515         struct nft_trans *trans;
 516         int err;
 517
 518         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 519         if (trans == NULL)
 520                 return -ENOMEM;
 521
 522         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 523                 flow = nft_flow_rule_create(ctx->net, rule);
 524                 if (IS_ERR(flow)) {
 525                         nft_trans_destroy(trans);
 526                         return PTR_ERR(flow);
 527                 }
 528
 529                 nft_trans_flow_rule(trans) = flow;
 530         }
 531
 532         err = nf_tables_delrule_deactivate(ctx, rule);
 533         if (err < 0) {
 534                 nft_trans_destroy(trans);
 535                 return err;
 536         }
 537         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 538
 539         return 0;
 540 }
 541
 542 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 543 {
 544         struct nft_rule *rule;
 545         int err;
 546
 547         list_for_each_entry(rule, &ctx->chain->rules, list) {
 548                 if (!nft_is_active_next(ctx->net, rule))
 549                         continue;
 550
 551                 err = nft_delrule(ctx, rule);
 552                 if (err < 0)
 553                         return err;
 554         }
 555         return 0;
 556 }
 557
 558 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 559                                struct nft_set *set,
 560                                const struct nft_set_desc *desc)
 561 {
 562         struct nft_trans *trans;
 563
 564         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 565         if (trans == NULL)
 566                 return -ENOMEM;
 567
 568         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 569                 nft_trans_set_id(trans) =
 570                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 571                 nft_activate_next(ctx->net, set);
 572         }
 573         nft_trans_set(trans) = set;
 574         if (desc) {
 575                 nft_trans_set_update(trans) = true;
 576                 nft_trans_set_gc_int(trans) = desc->gc_int;
 577                 nft_trans_set_timeout(trans) = desc->timeout;
 578                 nft_trans_set_size(trans) = desc->size;
 579         }
 580         nft_trans_commit_list_add_tail(ctx->net, trans);
 581
 582         return 0;
 583 }
 584
 585 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 586                              struct nft_set *set)
 587 {
 588         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 589 }
 590
 591 static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
 592                                   struct nft_set *set,
 593                                   const struct nft_set_iter *iter,
 594                                   struct nft_elem_priv *elem_priv)
 595 {
 596         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
 597
 598         return 0;
 599 }
 600
 601 struct nft_set_elem_catchall {
 602         struct list_head        list;
 603         struct rcu_head         rcu;
 604         struct nft_elem_priv    *elem;
 605 };
 606
 607 static void nft_map_catchall_deactivate(const struct nft_ctx *ctx,
 608                                         struct nft_set *set)
 609 {
 610         u8 genmask = nft_genmask_next(ctx->net);
 611         struct nft_set_elem_catchall *catchall;
 612         struct nft_set_ext *ext;
 613
 614         list_for_each_entry(catchall, &set->catchall_list, list) {
 615                 ext = nft_set_elem_ext(set, catchall->elem);
 616                 if (!nft_set_elem_active(ext, genmask))
 617                         continue;
 618
 619                 nft_setelem_data_deactivate(ctx->net, set, catchall->elem);
 620                 break;
 621         }
 622 }
 623
 624 static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set)
 625 {
 626         struct nft_set_iter iter = {
 627                 .genmask        = nft_genmask_next(ctx->net),
 628                 .fn             = nft_mapelem_deactivate,
 629         };
 630
 631         set->ops->walk(ctx, set, &iter);
 632         WARN_ON_ONCE(iter.err);
 633
 634         nft_map_catchall_deactivate(ctx, set);
 635 }
 636
 637 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 638 {
 639         int err;
 640
 641         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 642         if (err < 0)
 643                 return err;
 644
 645         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 646                 nft_map_deactivate(ctx, set);
 647
 648         nft_deactivate_next(ctx->net, set);
 649         nft_use_dec(&ctx->table->use);
 650
 651         return err;
 652 }
 653
 654 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 655                              struct nft_object *obj)
 656 {
 657         struct nft_trans *trans;
 658
 659         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 660         if (trans == NULL)
 661                 return -ENOMEM;
 662
 663         if (msg_type == NFT_MSG_NEWOBJ)
 664                 nft_activate_next(ctx->net, obj);
 665
 666         nft_trans_obj(trans) = obj;
 667         nft_trans_commit_list_add_tail(ctx->net, trans);
 668
 669         return 0;
 670 }
 671
 672 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 673 {
 674         int err;
 675
 676         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 677         if (err < 0)
 678                 return err;
 679
 680         nft_deactivate_next(ctx->net, obj);
 681         nft_use_dec(&ctx->table->use);
 682
 683         return err;
 684 }
 685
 686 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 687                                    struct nft_flowtable *flowtable)
 688 {
 689         struct nft_trans *trans;
 690
 691         trans = nft_trans_alloc(ctx, msg_type,
 692                                 sizeof(struct nft_trans_flowtable));
 693         if (trans == NULL)
 694                 return -ENOMEM;
 695
 696         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 697                 nft_activate_next(ctx->net, flowtable);
 698
 699         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 700         nft_trans_flowtable(trans) = flowtable;
 701         nft_trans_commit_list_add_tail(ctx->net, trans);
 702
 703         return 0;
 704 }
 705
 706 static int nft_delflowtable(struct nft_ctx *ctx,
 707                             struct nft_flowtable *flowtable)
 708 {
 709         int err;
 710
 711         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 712         if (err < 0)
 713                 return err;
 714
 715         nft_deactivate_next(ctx->net, flowtable);
 716         nft_use_dec(&ctx->table->use);
 717
 718         return err;
 719 }
 720
 721 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 722 {
 723         int i;
 724
 725         for (i = track->regs[dreg].num_reg; i > 0; i--)
 726                 __nft_reg_track_cancel(track, dreg - i);
 727 }
 728
 729 static void __nft_reg_track_update(struct nft_regs_track *track,
 730                                    const struct nft_expr *expr,
 731                                    u8 dreg, u8 num_reg)
 732 {
 733         track->regs[dreg].selector = expr;
 734         track->regs[dreg].bitwise = NULL;
 735         track->regs[dreg].num_reg = num_reg;
 736 }
 737
 738 void nft_reg_track_update(struct nft_regs_track *track,
 739                           const struct nft_expr *expr, u8 dreg, u8 len)
 740 {
 741         unsigned int regcount;
 742         int i;
 743
 744         __nft_reg_track_clobber(track, dreg);
 745
 746         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 747         for (i = 0; i < regcount; i++, dreg++)
 748                 __nft_reg_track_update(track, expr, dreg, i);
 749 }
 750 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 751
 752 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 753 {
 754         unsigned int regcount;
 755         int i;
 756
 757         __nft_reg_track_clobber(track, dreg);
 758
 759         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 760         for (i = 0; i < regcount; i++, dreg++)
 761                 __nft_reg_track_cancel(track, dreg);
 762 }
 763 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 764
 765 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 766 {
 767         track->regs[dreg].selector = NULL;
 768         track->regs[dreg].bitwise = NULL;
 769         track->regs[dreg].num_reg = 0;
 770 }
 771 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 772
 773 /*
 774  * Tables
 775  */
 776
 777 static struct nft_table *nft_table_lookup(const struct net *net,
 778                                           const struct nlattr *nla,
 779                                           u8 family, u8 genmask, u32 nlpid)
 780 {
 781         struct nftables_pernet *nft_net;
 782         struct nft_table *table;
 783
 784         if (nla == NULL)
 785                 return ERR_PTR(-EINVAL);
 786
 787         nft_net = nft_pernet(net);
 788         list_for_each_entry_rcu(table, &nft_net->tables, list,
 789                                 lockdep_is_held(&nft_net->commit_mutex)) {
 790                 if (!nla_strcmp(nla, table->name) &&
 791                     table->family == family &&
 792                     nft_active_genmask(table, genmask)) {
 793                         if (nft_table_has_owner(table) &&
 794                             nlpid && table->nlpid != nlpid)
 795                                 return ERR_PTR(-EPERM);
 796
 797                         return table;
 798                 }
 799         }
 800
 801         return ERR_PTR(-ENOENT);
 802 }
 803
 804 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 805                                                    const struct nlattr *nla,
 806                                                    int family, u8 genmask, u32 nlpid)
 807 {
 808         struct nftables_pernet *nft_net;
 809         struct nft_table *table;
 810
 811         nft_net = nft_pernet(net);
 812         list_for_each_entry(table, &nft_net->tables, list) {
 813                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 814                     table->family == family &&
 815                     nft_active_genmask(table, genmask)) {
 816                         if (nft_table_has_owner(table) &&
 817                             nlpid && table->nlpid != nlpid)
 818                                 return ERR_PTR(-EPERM);
 819
 820                         return table;
 821                 }
 822         }
 823
 824         return ERR_PTR(-ENOENT);
 825 }
 826
 827 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 828 {
 829         return ++table->hgenerator;
 830 }
 831
 832 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 833
 834 static const struct nft_chain_type *
 835 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 836 {
 837         if (family >= NFPROTO_NUMPROTO ||
 838             type >= NFT_CHAIN_T_MAX)
 839                 return NULL;
 840
 841         return chain_type[family][type];
 842 }
 843
 844 static const struct nft_chain_type *
 845 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 846 {
 847         const struct nft_chain_type *type;
 848         int i;
 849
 850         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 851                 type = __nft_chain_type_get(family, i);
 852                 if (!type)
 853                         continue;
 854                 if (!nla_strcmp(nla, type->name))
 855                         return type;
 856         }
 857         return NULL;
 858 }
 859
 860 struct nft_module_request {
 861         struct list_head        list;
 862         char                    module[MODULE_NAME_LEN];
 863         bool                    done;
 864 };
 865
 866 #ifdef CONFIG_MODULES
 867 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 868                                       ...)
 869 {
 870         char module_name[MODULE_NAME_LEN];
 871         struct nftables_pernet *nft_net;
 872         struct nft_module_request *req;
 873         va_list args;
 874         int ret;
 875
 876         va_start(args, fmt);
 877         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 878         va_end(args);
 879         if (ret >= MODULE_NAME_LEN)
 880                 return 0;
 881
 882         nft_net = nft_pernet(net);
 883         list_for_each_entry(req, &nft_net->module_list, list) {
 884                 if (!strcmp(req->module, module_name)) {
 885                         if (req->done)
 886                                 return 0;
 887
 888                         /* A request to load this module already exists. */
 889                         return -EAGAIN;
 890                 }
 891         }
 892
 893         req = kmalloc(sizeof(*req), GFP_KERNEL);
 894         if (!req)
 895                 return -ENOMEM;
 896
 897         req->done = false;
 898         strscpy(req->module, module_name, MODULE_NAME_LEN);
 899         list_add_tail(&req->list, &nft_net->module_list);
 900
 901         return -EAGAIN;
 902 }
 903 EXPORT_SYMBOL_GPL(nft_request_module);
 904 #endif
 905
 906 static void lockdep_nfnl_nft_mutex_not_held(void)
 907 {
 908 #ifdef CONFIG_PROVE_LOCKING
 909         if (debug_locks)
 910                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 911 #endif
 912 }
 913
 914 static const struct nft_chain_type *
 915 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 916                             u8 family, bool autoload)
 917 {
 918         const struct nft_chain_type *type;
 919
 920         type = __nf_tables_chain_type_lookup(nla, family);
 921         if (type != NULL)
 922                 return type;
 923
 924         lockdep_nfnl_nft_mutex_not_held();
 925 #ifdef CONFIG_MODULES
 926         if (autoload) {
 927                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 928                                        nla_len(nla),
 929                                        (const char *)nla_data(nla)) == -EAGAIN)
 930                         return ERR_PTR(-EAGAIN);
 931         }
 932 #endif
 933         return ERR_PTR(-ENOENT);
 934 }
 935
 936 static __be16 nft_base_seq(const struct net *net)
 937 {
 938         struct nftables_pernet *nft_net = nft_pernet(net);
 939
 940         return htons(nft_net->base_seq & 0xffff);
 941 }
 942
 943 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 944         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 945                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 946         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 947         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 948         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 949                                     .len = NFT_USERDATA_MAXLEN }
 950 };
 951
 952 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 953                                      u32 portid, u32 seq, int event, u32 flags,
 954                                      int family, const struct nft_table *table)
 955 {
 956         struct nlmsghdr *nlh;
 957
 958         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 959         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 960                            NFNETLINK_V0, nft_base_seq(net));
 961         if (!nlh)
 962                 goto nla_put_failure;
 963
 964         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 965             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 966             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 967                          NFTA_TABLE_PAD))
 968                 goto nla_put_failure;
 969
 970         if (event == NFT_MSG_DELTABLE) {
 971                 nlmsg_end(skb, nlh);
 972                 return 0;
 973         }
 974
 975         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 976                          htonl(table->flags & NFT_TABLE_F_MASK)))
 977                 goto nla_put_failure;
 978
 979         if (nft_table_has_owner(table) &&
 980             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 981                 goto nla_put_failure;
 982
 983         if (table->udata) {
 984                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 985                         goto nla_put_failure;
 986         }
 987
 988         nlmsg_end(skb, nlh);
 989         return 0;
 990
 991 nla_put_failure:
 992         nlmsg_trim(skb, nlh);
 993         return -1;
 994 }
 995
 996 struct nftnl_skb_parms {
 997         bool report;
 998 };
 999 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
1000
1001 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
1002                                struct list_head *notify_list)
1003 {
1004         NFT_CB(skb).report = report;
1005         list_add_tail(&skb->list, notify_list);
1006 }
1007
1008 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
1009 {
1010         struct nftables_pernet *nft_net;
1011         struct sk_buff *skb;
1012         u16 flags = 0;
1013         int err;
1014
1015         if (!ctx->report &&
1016             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1017                 return;
1018
1019         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1020         if (skb == NULL)
1021                 goto err;
1022
1023         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1024                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1025
1026         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
1027                                         event, flags, ctx->family, ctx->table);
1028         if (err < 0) {
1029                 kfree_skb(skb);
1030                 goto err;
1031         }
1032
1033         nft_net = nft_pernet(ctx->net);
1034         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1035         return;
1036 err:
1037         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1038 }
1039
1040 static int nf_tables_dump_tables(struct sk_buff *skb,
1041                                  struct netlink_callback *cb)
1042 {
1043         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1044         struct nftables_pernet *nft_net;
1045         const struct nft_table *table;
1046         unsigned int idx = 0, s_idx = cb->args[0];
1047         struct net *net = sock_net(skb->sk);
1048         int family = nfmsg->nfgen_family;
1049
1050         rcu_read_lock();
1051         nft_net = nft_pernet(net);
1052         cb->seq = READ_ONCE(nft_net->base_seq);
1053
1054         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1055                 if (family != NFPROTO_UNSPEC && family != table->family)
1056                         continue;
1057
1058                 if (idx < s_idx)
1059                         goto cont;
1060                 if (idx > s_idx)
1061                         memset(&cb->args[1], 0,
1062                                sizeof(cb->args) - sizeof(cb->args[0]));
1063                 if (!nft_is_active(net, table))
1064                         continue;
1065                 if (nf_tables_fill_table_info(skb, net,
1066                                               NETLINK_CB(cb->skb).portid,
1067                                               cb->nlh->nlmsg_seq,
1068                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
1069                                               table->family, table) < 0)
1070                         goto done;
1071
1072                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1073 cont:
1074                 idx++;
1075         }
1076 done:
1077         rcu_read_unlock();
1078         cb->args[0] = idx;
1079         return skb->len;
1080 }
1081
1082 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
1083                                       const struct nlmsghdr *nlh,
1084                                       struct netlink_dump_control *c)
1085 {
1086         int err;
1087
1088         if (!try_module_get(THIS_MODULE))
1089                 return -EINVAL;
1090
1091         rcu_read_unlock();
1092         err = netlink_dump_start(nlsk, skb, nlh, c);
1093         rcu_read_lock();
1094         module_put(THIS_MODULE);
1095
1096         return err;
1097 }
1098
1099 /* called with rcu_read_lock held */
1100 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
1101                               const struct nlattr * const nla[])
1102 {
1103         struct netlink_ext_ack *extack = info->extack;
1104         u8 genmask = nft_genmask_cur(info->net);
1105         u8 family = info->nfmsg->nfgen_family;
1106         const struct nft_table *table;
1107         struct net *net = info->net;
1108         struct sk_buff *skb2;
1109         int err;
1110
1111         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1112                 struct netlink_dump_control c = {
1113                         .dump = nf_tables_dump_tables,
1114                         .module = THIS_MODULE,
1115                 };
1116
1117                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1118         }
1119
1120         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
1121         if (IS_ERR(table)) {
1122                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
1123                 return PTR_ERR(table);
1124         }
1125
1126         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1127         if (!skb2)
1128                 return -ENOMEM;
1129
1130         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
1131                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
1132                                         0, family, table);
1133         if (err < 0)
1134                 goto err_fill_table_info;
1135
1136         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1137
1138 err_fill_table_info:
1139         kfree_skb(skb2);
1140         return err;
1141 }
1142
1143 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1144 {
1145         struct nft_chain *chain;
1146         u32 i = 0;
1147
1148         list_for_each_entry(chain, &table->chains, list) {
1149                 if (!nft_is_active_next(net, chain))
1150                         continue;
1151                 if (!nft_is_base_chain(chain))
1152                         continue;
1153
1154                 if (cnt && i++ == cnt)
1155                         break;
1156
1157                 nf_tables_unregister_hook(net, table, chain);
1158         }
1159 }
1160
1161 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1162 {
1163         struct nft_chain *chain;
1164         int err, i = 0;
1165
1166         list_for_each_entry(chain, &table->chains, list) {
1167                 if (!nft_is_active_next(net, chain))
1168                         continue;
1169                 if (!nft_is_base_chain(chain))
1170                         continue;
1171
1172                 err = nf_tables_register_hook(net, table, chain);
1173                 if (err < 0)
1174                         goto err_register_hooks;
1175
1176                 i++;
1177         }
1178         return 0;
1179
1180 err_register_hooks:
1181         if (i)
1182                 nft_table_disable(net, table, i);
1183         return err;
1184 }
1185
1186 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1187 {
1188         table->flags &= ~NFT_TABLE_F_DORMANT;
1189         nft_table_disable(net, table, 0);
1190         table->flags |= NFT_TABLE_F_DORMANT;
1191 }
1192
1193 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1194 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1195 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1196 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1197                                          __NFT_TABLE_F_WAS_AWAKEN)
1198
1199 static int nf_tables_updtable(struct nft_ctx *ctx)
1200 {
1201         struct nft_trans *trans;
1202         u32 flags;
1203         int ret;
1204
1205         if (!ctx->nla[NFTA_TABLE_FLAGS])
1206                 return 0;
1207
1208         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1209         if (flags & ~NFT_TABLE_F_MASK)
1210                 return -EOPNOTSUPP;
1211
1212         if (flags == ctx->table->flags)
1213                 return 0;
1214
1215         if ((nft_table_has_owner(ctx->table) &&
1216              !(flags & NFT_TABLE_F_OWNER)) ||
1217             (!nft_table_has_owner(ctx->table) &&
1218              flags & NFT_TABLE_F_OWNER))
1219                 return -EOPNOTSUPP;
1220
1221         /* No dormant off/on/off/on games in single transaction */
1222         if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
1223                 return -EINVAL;
1224
1225         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1226                                 sizeof(struct nft_trans_table));
1227         if (trans == NULL)
1228                 return -ENOMEM;
1229
1230         if ((flags & NFT_TABLE_F_DORMANT) &&
1231             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1232                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1233                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1234                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1235         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1236                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1237                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1238                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1239                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1240                         if (ret < 0)
1241                                 goto err_register_hooks;
1242
1243                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1244                 }
1245         }
1246
1247         nft_trans_table_update(trans) = true;
1248         nft_trans_commit_list_add_tail(ctx->net, trans);
1249
1250         return 0;
1251
1252 err_register_hooks:
1253         nft_trans_destroy(trans);
1254         return ret;
1255 }
1256
1257 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1258 {
1259         const char *name = data;
1260
1261         return jhash(name, strlen(name), seed);
1262 }
1263
1264 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1265 {
1266         const struct nft_chain *chain = data;
1267
1268         return nft_chain_hash(chain->name, 0, seed);
1269 }
1270
1271 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1272                               const void *ptr)
1273 {
1274         const struct nft_chain *chain = ptr;
1275         const char *name = arg->key;
1276
1277         return strcmp(chain->name, name);
1278 }
1279
1280 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1281 {
1282         const struct nft_object_hash_key *k = data;
1283
1284         seed ^= hash_ptr(k->table, 32);
1285
1286         return jhash(k->name, strlen(k->name), seed);
1287 }
1288
1289 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1290 {
1291         const struct nft_object *obj = data;
1292
1293         return nft_objname_hash(&obj->key, 0, seed);
1294 }
1295
1296 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1297                                 const void *ptr)
1298 {
1299         const struct nft_object_hash_key *k = arg->key;
1300         const struct nft_object *obj = ptr;
1301
1302         if (obj->key.table != k->table)
1303                 return -1;
1304
1305         return strcmp(obj->key.name, k->name);
1306 }
1307
1308 static bool nft_supported_family(u8 family)
1309 {
1310         return false
1311 #ifdef CONFIG_NF_TABLES_INET
1312                 || family == NFPROTO_INET
1313 #endif
1314 #ifdef CONFIG_NF_TABLES_IPV4
1315                 || family == NFPROTO_IPV4
1316 #endif
1317 #ifdef CONFIG_NF_TABLES_ARP
1318                 || family == NFPROTO_ARP
1319 #endif
1320 #ifdef CONFIG_NF_TABLES_NETDEV
1321                 || family == NFPROTO_NETDEV
1322 #endif
1323 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1324                 || family == NFPROTO_BRIDGE
1325 #endif
1326 #ifdef CONFIG_NF_TABLES_IPV6
1327                 || family == NFPROTO_IPV6
1328 #endif
1329                 ;
1330 }
1331
1332 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1333                               const struct nlattr * const nla[])
1334 {
1335         struct nftables_pernet *nft_net = nft_pernet(info->net);
1336         struct netlink_ext_ack *extack = info->extack;
1337         u8 genmask = nft_genmask_next(info->net);
1338         u8 family = info->nfmsg->nfgen_family;
1339         struct net *net = info->net;
1340         const struct nlattr *attr;
1341         struct nft_table *table;
1342         struct nft_ctx ctx;
1343         u32 flags = 0;
1344         int err;
1345
1346         if (!nft_supported_family(family))
1347                 return -EOPNOTSUPP;
1348
1349         lockdep_assert_held(&nft_net->commit_mutex);
1350         attr = nla[NFTA_TABLE_NAME];
1351         table = nft_table_lookup(net, attr, family, genmask,
1352                                  NETLINK_CB(skb).portid);
1353         if (IS_ERR(table)) {
1354                 if (PTR_ERR(table) != -ENOENT)
1355                         return PTR_ERR(table);
1356         } else {
1357                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1358                         NL_SET_BAD_ATTR(extack, attr);
1359                         return -EEXIST;
1360                 }
1361                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1362                         return -EOPNOTSUPP;
1363
1364                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1365
1366                 return nf_tables_updtable(&ctx);
1367         }
1368
1369         if (nla[NFTA_TABLE_FLAGS]) {
1370                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1371                 if (flags & ~NFT_TABLE_F_MASK)
1372                         return -EOPNOTSUPP;
1373         }
1374
1375         err = -ENOMEM;
1376         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1377         if (table == NULL)
1378                 goto err_kzalloc;
1379
1380         table->validate_state = nft_net->validate_state;
1381         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1382         if (table->name == NULL)
1383                 goto err_strdup;
1384
1385         if (nla[NFTA_TABLE_USERDATA]) {
1386                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1387                 if (table->udata == NULL)
1388                         goto err_table_udata;
1389
1390                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1391         }
1392
1393         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1394         if (err)
1395                 goto err_chain_ht;
1396
1397         INIT_LIST_HEAD(&table->chains);
1398         INIT_LIST_HEAD(&table->sets);
1399         INIT_LIST_HEAD(&table->objects);
1400         INIT_LIST_HEAD(&table->flowtables);
1401         table->family = family;
1402         table->flags = flags;
1403         table->handle = ++nft_net->table_handle;
1404         if (table->flags & NFT_TABLE_F_OWNER)
1405                 table->nlpid = NETLINK_CB(skb).portid;
1406
1407         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1408         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1409         if (err < 0)
1410                 goto err_trans;
1411
1412         list_add_tail_rcu(&table->list, &nft_net->tables);
1413         return 0;
1414 err_trans:
1415         rhltable_destroy(&table->chains_ht);
1416 err_chain_ht:
1417         kfree(table->udata);
1418 err_table_udata:
1419         kfree(table->name);
1420 err_strdup:
1421         kfree(table);
1422 err_kzalloc:
1423         return err;
1424 }
1425
1426 static int nft_flush_table(struct nft_ctx *ctx)
1427 {
1428         struct nft_flowtable *flowtable, *nft;
1429         struct nft_chain *chain, *nc;
1430         struct nft_object *obj, *ne;
1431         struct nft_set *set, *ns;
1432         int err;
1433
1434         list_for_each_entry(chain, &ctx->table->chains, list) {
1435                 if (!nft_is_active_next(ctx->net, chain))
1436                         continue;
1437
1438                 if (nft_chain_binding(chain))
1439                         continue;
1440
1441                 ctx->chain = chain;
1442
1443                 err = nft_delrule_by_chain(ctx);
1444                 if (err < 0)
1445                         goto out;
1446         }
1447
1448         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1449                 if (!nft_is_active_next(ctx->net, set))
1450                         continue;
1451
1452                 if (nft_set_is_anonymous(set))
1453                         continue;
1454
1455                 err = nft_delset(ctx, set);
1456                 if (err < 0)
1457                         goto out;
1458         }
1459
1460         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1461                 if (!nft_is_active_next(ctx->net, flowtable))
1462                         continue;
1463
1464                 err = nft_delflowtable(ctx, flowtable);
1465                 if (err < 0)
1466                         goto out;
1467         }
1468
1469         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1470                 if (!nft_is_active_next(ctx->net, obj))
1471                         continue;
1472
1473                 err = nft_delobj(ctx, obj);
1474                 if (err < 0)
1475                         goto out;
1476         }
1477
1478         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1479                 if (!nft_is_active_next(ctx->net, chain))
1480                         continue;
1481
1482                 if (nft_chain_binding(chain))
1483                         continue;
1484
1485                 ctx->chain = chain;
1486
1487                 err = nft_delchain(ctx);
1488                 if (err < 0)
1489                         goto out;
1490         }
1491
1492         err = nft_deltable(ctx);
1493 out:
1494         return err;
1495 }
1496
1497 static int nft_flush(struct nft_ctx *ctx, int family)
1498 {
1499         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1500         const struct nlattr * const *nla = ctx->nla;
1501         struct nft_table *table, *nt;
1502         int err = 0;
1503
1504         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1505                 if (family != AF_UNSPEC && table->family != family)
1506                         continue;
1507
1508                 ctx->family = table->family;
1509
1510                 if (!nft_is_active_next(ctx->net, table))
1511                         continue;
1512
1513                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1514                         continue;
1515
1516                 if (nla[NFTA_TABLE_NAME] &&
1517                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1518                         continue;
1519
1520                 ctx->table = table;
1521
1522                 err = nft_flush_table(ctx);
1523                 if (err < 0)
1524                         goto out;
1525         }
1526 out:
1527         return err;
1528 }
1529
1530 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1531                               const struct nlattr * const nla[])
1532 {
1533         struct netlink_ext_ack *extack = info->extack;
1534         u8 genmask = nft_genmask_next(info->net);
1535         u8 family = info->nfmsg->nfgen_family;
1536         struct net *net = info->net;
1537         const struct nlattr *attr;
1538         struct nft_table *table;
1539         struct nft_ctx ctx;
1540
1541         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1542         if (family == AF_UNSPEC ||
1543             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1544                 return nft_flush(&ctx, family);
1545
1546         if (nla[NFTA_TABLE_HANDLE]) {
1547                 attr = nla[NFTA_TABLE_HANDLE];
1548                 table = nft_table_lookup_byhandle(net, attr, family, genmask,
1549                                                   NETLINK_CB(skb).portid);
1550         } else {
1551                 attr = nla[NFTA_TABLE_NAME];
1552                 table = nft_table_lookup(net, attr, family, genmask,
1553                                          NETLINK_CB(skb).portid);
1554         }
1555
1556         if (IS_ERR(table)) {
1557                 if (PTR_ERR(table) == -ENOENT &&
1558                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1559                         return 0;
1560
1561                 NL_SET_BAD_ATTR(extack, attr);
1562                 return PTR_ERR(table);
1563         }
1564
1565         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1566             table->use > 0)
1567                 return -EBUSY;
1568
1569         ctx.family = family;
1570         ctx.table = table;
1571
1572         return nft_flush_table(&ctx);
1573 }
1574
1575 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1576 {
1577         if (WARN_ON(ctx->table->use > 0))
1578                 return;
1579
1580         rhltable_destroy(&ctx->table->chains_ht);
1581         kfree(ctx->table->name);
1582         kfree(ctx->table->udata);
1583         kfree(ctx->table);
1584 }
1585
1586 void nft_register_chain_type(const struct nft_chain_type *ctype)
1587 {
1588         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1589         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1590                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1591                 return;
1592         }
1593         chain_type[ctype->family][ctype->type] = ctype;
1594         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1595 }
1596 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1597
1598 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1599 {
1600         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1601         chain_type[ctype->family][ctype->type] = NULL;
1602         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1603 }
1604 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1605
1606 /*
1607  * Chains
1608  */
1609
1610 static struct nft_chain *
1611 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1612 {
1613         struct nft_chain *chain;
1614
1615         list_for_each_entry(chain, &table->chains, list) {
1616                 if (chain->handle == handle &&
1617                     nft_active_genmask(chain, genmask))
1618                         return chain;
1619         }
1620
1621         return ERR_PTR(-ENOENT);
1622 }
1623
1624 static bool lockdep_commit_lock_is_held(const struct net *net)
1625 {
1626 #ifdef CONFIG_PROVE_LOCKING
1627         struct nftables_pernet *nft_net = nft_pernet(net);
1628
1629         return lockdep_is_held(&nft_net->commit_mutex);
1630 #else
1631         return true;
1632 #endif
1633 }
1634
1635 static struct nft_chain *nft_chain_lookup(struct net *net,
1636                                           struct nft_table *table,
1637                                           const struct nlattr *nla, u8 genmask)
1638 {
1639         char search[NFT_CHAIN_MAXNAMELEN + 1];
1640         struct rhlist_head *tmp, *list;
1641         struct nft_chain *chain;
1642
1643         if (nla == NULL)
1644                 return ERR_PTR(-EINVAL);
1645
1646         nla_strscpy(search, nla, sizeof(search));
1647
1648         WARN_ON(!rcu_read_lock_held() &&
1649                 !lockdep_commit_lock_is_held(net));
1650
1651         chain = ERR_PTR(-ENOENT);
1652         rcu_read_lock();
1653         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1654         if (!list)
1655                 goto out_unlock;
1656
1657         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1658                 if (nft_active_genmask(chain, genmask))
1659                         goto out_unlock;
1660         }
1661         chain = ERR_PTR(-ENOENT);
1662 out_unlock:
1663         rcu_read_unlock();
1664         return chain;
1665 }
1666
1667 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1668         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1669                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1670         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1671         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1672                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1673         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1674         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1675         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1676                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1677         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1678         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1679         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1680         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1681                                     .len = NFT_USERDATA_MAXLEN },
1682 };
1683
1684 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1685         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1686         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1687         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1688                                     .len = IFNAMSIZ - 1 },
1689 };
1690
1691 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1692 {
1693         struct nft_stats *cpu_stats, total;
1694         struct nlattr *nest;
1695         unsigned int seq;
1696         u64 pkts, bytes;
1697         int cpu;
1698
1699         if (!stats)
1700                 return 0;
1701
1702         memset(&total, 0, sizeof(total));
1703         for_each_possible_cpu(cpu) {
1704                 cpu_stats = per_cpu_ptr(stats, cpu);
1705                 do {
1706                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1707                         pkts = cpu_stats->pkts;
1708                         bytes = cpu_stats->bytes;
1709                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1710                 total.pkts += pkts;
1711                 total.bytes += bytes;
1712         }
1713         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1714         if (nest == NULL)
1715                 goto nla_put_failure;
1716
1717         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1718                          NFTA_COUNTER_PAD) ||
1719             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1720                          NFTA_COUNTER_PAD))
1721                 goto nla_put_failure;
1722
1723         nla_nest_end(skb, nest);
1724         return 0;
1725
1726 nla_put_failure:
1727         return -ENOSPC;
1728 }
1729
1730 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1731                                    const struct nft_base_chain *basechain,
1732                                    const struct list_head *hook_list)
1733 {
1734         const struct nf_hook_ops *ops = &basechain->ops;
1735         struct nft_hook *hook, *first = NULL;
1736         struct nlattr *nest, *nest_devs;
1737         int n = 0;
1738
1739         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1740         if (nest == NULL)
1741                 goto nla_put_failure;
1742         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1743                 goto nla_put_failure;
1744         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1745                 goto nla_put_failure;
1746
1747         if (nft_base_chain_netdev(family, ops->hooknum)) {
1748                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1749                 if (!nest_devs)
1750                         goto nla_put_failure;
1751
1752                 if (!hook_list)
1753                         hook_list = &basechain->hook_list;
1754
1755                 list_for_each_entry(hook, hook_list, list) {
1756                         if (!first)
1757                                 first = hook;
1758
1759                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1760                                            hook->ops.dev->name))
1761                                 goto nla_put_failure;
1762                         n++;
1763                 }
1764                 nla_nest_end(skb, nest_devs);
1765
1766                 if (n == 1 &&
1767                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1768                         goto nla_put_failure;
1769         }
1770         nla_nest_end(skb, nest);
1771
1772         return 0;
1773 nla_put_failure:
1774         return -1;
1775 }
1776
1777 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1778                                      u32 portid, u32 seq, int event, u32 flags,
1779                                      int family, const struct nft_table *table,
1780                                      const struct nft_chain *chain,
1781                                      const struct list_head *hook_list)
1782 {
1783         struct nlmsghdr *nlh;
1784
1785         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1786         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1787                            NFNETLINK_V0, nft_base_seq(net));
1788         if (!nlh)
1789                 goto nla_put_failure;
1790
1791         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1792             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1793             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1794                          NFTA_CHAIN_PAD))
1795                 goto nla_put_failure;
1796
1797         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1798                 nlmsg_end(skb, nlh);
1799                 return 0;
1800         }
1801
1802         if (nft_is_base_chain(chain)) {
1803                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1804                 struct nft_stats __percpu *stats;
1805
1806                 if (nft_dump_basechain_hook(skb, family, basechain, hook_list))
1807                         goto nla_put_failure;
1808
1809                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1810                                  htonl(basechain->policy)))
1811                         goto nla_put_failure;
1812
1813                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1814                         goto nla_put_failure;
1815
1816                 stats = rcu_dereference_check(basechain->stats,
1817                                               lockdep_commit_lock_is_held(net));
1818                 if (nft_dump_stats(skb, stats))
1819                         goto nla_put_failure;
1820         }
1821
1822         if (chain->flags &&
1823             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1824                 goto nla_put_failure;
1825
1826         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1827                 goto nla_put_failure;
1828
1829         if (chain->udata &&
1830             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1831                 goto nla_put_failure;
1832
1833         nlmsg_end(skb, nlh);
1834         return 0;
1835
1836 nla_put_failure:
1837         nlmsg_trim(skb, nlh);
1838         return -1;
1839 }
1840
1841 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
1842                                    const struct list_head *hook_list)
1843 {
1844         struct nftables_pernet *nft_net;
1845         struct sk_buff *skb;
1846         u16 flags = 0;
1847         int err;
1848
1849         if (!ctx->report &&
1850             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1851                 return;
1852
1853         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1854         if (skb == NULL)
1855                 goto err;
1856
1857         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1858                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1859
1860         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1861                                         event, flags, ctx->family, ctx->table,
1862                                         ctx->chain, hook_list);
1863         if (err < 0) {
1864                 kfree_skb(skb);
1865                 goto err;
1866         }
1867
1868         nft_net = nft_pernet(ctx->net);
1869         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1870         return;
1871 err:
1872         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1873 }
1874
1875 static int nf_tables_dump_chains(struct sk_buff *skb,
1876                                  struct netlink_callback *cb)
1877 {
1878         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1879         unsigned int idx = 0, s_idx = cb->args[0];
1880         struct net *net = sock_net(skb->sk);
1881         int family = nfmsg->nfgen_family;
1882         struct nftables_pernet *nft_net;
1883         const struct nft_table *table;
1884         const struct nft_chain *chain;
1885
1886         rcu_read_lock();
1887         nft_net = nft_pernet(net);
1888         cb->seq = READ_ONCE(nft_net->base_seq);
1889
1890         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1891                 if (family != NFPROTO_UNSPEC && family != table->family)
1892                         continue;
1893
1894                 list_for_each_entry_rcu(chain, &table->chains, list) {
1895                         if (idx < s_idx)
1896                                 goto cont;
1897                         if (idx > s_idx)
1898                                 memset(&cb->args[1], 0,
1899                                        sizeof(cb->args) - sizeof(cb->args[0]));
1900                         if (!nft_is_active(net, chain))
1901                                 continue;
1902                         if (nf_tables_fill_chain_info(skb, net,
1903                                                       NETLINK_CB(cb->skb).portid,
1904                                                       cb->nlh->nlmsg_seq,
1905                                                       NFT_MSG_NEWCHAIN,
1906                                                       NLM_F_MULTI,
1907                                                       table->family, table,
1908                                                       chain, NULL) < 0)
1909                                 goto done;
1910
1911                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1912 cont:
1913                         idx++;
1914                 }
1915         }
1916 done:
1917         rcu_read_unlock();
1918         cb->args[0] = idx;
1919         return skb->len;
1920 }
1921
1922 /* called with rcu_read_lock held */
1923 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1924                               const struct nlattr * const nla[])
1925 {
1926         struct netlink_ext_ack *extack = info->extack;
1927         u8 genmask = nft_genmask_cur(info->net);
1928         u8 family = info->nfmsg->nfgen_family;
1929         const struct nft_chain *chain;
1930         struct net *net = info->net;
1931         struct nft_table *table;
1932         struct sk_buff *skb2;
1933         int err;
1934
1935         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1936                 struct netlink_dump_control c = {
1937                         .dump = nf_tables_dump_chains,
1938                         .module = THIS_MODULE,
1939                 };
1940
1941                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1942         }
1943
1944         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1945         if (IS_ERR(table)) {
1946                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1947                 return PTR_ERR(table);
1948         }
1949
1950         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1951         if (IS_ERR(chain)) {
1952                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1953                 return PTR_ERR(chain);
1954         }
1955
1956         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1957         if (!skb2)
1958                 return -ENOMEM;
1959
1960         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1961                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
1962                                         0, family, table, chain, NULL);
1963         if (err < 0)
1964                 goto err_fill_chain_info;
1965
1966         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1967
1968 err_fill_chain_info:
1969         kfree_skb(skb2);
1970         return err;
1971 }
1972
1973 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1974         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1975         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1976 };
1977
1978 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1979 {
1980         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1981         struct nft_stats __percpu *newstats;
1982         struct nft_stats *stats;
1983         int err;
1984
1985         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1986                                           nft_counter_policy, NULL);
1987         if (err < 0)
1988                 return ERR_PTR(err);
1989
1990         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1991                 return ERR_PTR(-EINVAL);
1992
1993         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1994         if (newstats == NULL)
1995                 return ERR_PTR(-ENOMEM);
1996
1997         /* Restore old counters on this cpu, no problem. Per-cpu statistics
1998          * are not exposed to userspace.
1999          */
2000         preempt_disable();
2001         stats = this_cpu_ptr(newstats);
2002         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
2003         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
2004         preempt_enable();
2005
2006         return newstats;
2007 }
2008
2009 static void nft_chain_stats_replace(struct nft_trans *trans)
2010 {
2011         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
2012
2013         if (!nft_trans_chain_stats(trans))
2014                 return;
2015
2016         nft_trans_chain_stats(trans) =
2017                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
2018                                     lockdep_commit_lock_is_held(trans->ctx.net));
2019
2020         if (!nft_trans_chain_stats(trans))
2021                 static_branch_inc(&nft_counters_enabled);
2022 }
2023
2024 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
2025 {
2026         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
2027         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
2028
2029         if (g0 != g1)
2030                 kvfree(g1);
2031         kvfree(g0);
2032
2033         /* should be NULL either via abort or via successful commit */
2034         WARN_ON_ONCE(chain->blob_next);
2035         kvfree(chain->blob_next);
2036 }
2037
2038 void nf_tables_chain_destroy(struct nft_ctx *ctx)
2039 {
2040         struct nft_chain *chain = ctx->chain;
2041         struct nft_hook *hook, *next;
2042
2043         if (WARN_ON(chain->use > 0))
2044                 return;
2045
2046         /* no concurrent access possible anymore */
2047         nf_tables_chain_free_chain_rules(chain);
2048
2049         if (nft_is_base_chain(chain)) {
2050                 struct nft_base_chain *basechain = nft_base_chain(chain);
2051
2052                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2053                         list_for_each_entry_safe(hook, next,
2054                                                  &basechain->hook_list, list) {
2055                                 list_del_rcu(&hook->list);
2056                                 kfree_rcu(hook, rcu);
2057                         }
2058                 }
2059                 module_put(basechain->type->owner);
2060                 if (rcu_access_pointer(basechain->stats)) {
2061                         static_branch_dec(&nft_counters_enabled);
2062                         free_percpu(rcu_dereference_raw(basechain->stats));
2063                 }
2064                 kfree(chain->name);
2065                 kfree(chain->udata);
2066                 kfree(basechain);
2067         } else {
2068                 kfree(chain->name);
2069                 kfree(chain->udata);
2070                 kfree(chain);
2071         }
2072 }
2073
2074 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
2075                                               const struct nlattr *attr)
2076 {
2077         struct net_device *dev;
2078         char ifname[IFNAMSIZ];
2079         struct nft_hook *hook;
2080         int err;
2081
2082         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
2083         if (!hook) {
2084                 err = -ENOMEM;
2085                 goto err_hook_alloc;
2086         }
2087
2088         nla_strscpy(ifname, attr, IFNAMSIZ);
2089         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
2090          * indirectly serializing all the other holders of the commit_mutex with
2091          * the rtnl_mutex.
2092          */
2093         dev = __dev_get_by_name(net, ifname);
2094         if (!dev) {
2095                 err = -ENOENT;
2096                 goto err_hook_dev;
2097         }
2098         hook->ops.dev = dev;
2099
2100         return hook;
2101
2102 err_hook_dev:
2103         kfree(hook);
2104 err_hook_alloc:
2105         return ERR_PTR(err);
2106 }
2107
2108 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
2109                                            const struct nft_hook *this)
2110 {
2111         struct nft_hook *hook;
2112
2113         list_for_each_entry(hook, hook_list, list) {
2114                 if (this->ops.dev == hook->ops.dev)
2115                         return hook;
2116         }
2117
2118         return NULL;
2119 }
2120
2121 static int nf_tables_parse_netdev_hooks(struct net *net,
2122                                         const struct nlattr *attr,
2123                                         struct list_head *hook_list,
2124                                         struct netlink_ext_ack *extack)
2125 {
2126         struct nft_hook *hook, *next;
2127         const struct nlattr *tmp;
2128         int rem, n = 0, err;
2129
2130         nla_for_each_nested(tmp, attr, rem) {
2131                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
2132                         err = -EINVAL;
2133                         goto err_hook;
2134                 }
2135
2136                 hook = nft_netdev_hook_alloc(net, tmp);
2137                 if (IS_ERR(hook)) {
2138                         NL_SET_BAD_ATTR(extack, tmp);
2139                         err = PTR_ERR(hook);
2140                         goto err_hook;
2141                 }
2142                 if (nft_hook_list_find(hook_list, hook)) {
2143                         NL_SET_BAD_ATTR(extack, tmp);
2144                         kfree(hook);
2145                         err = -EEXIST;
2146                         goto err_hook;
2147                 }
2148                 list_add_tail(&hook->list, hook_list);
2149                 n++;
2150
2151                 if (n == NFT_NETDEVICE_MAX) {
2152                         err = -EFBIG;
2153                         goto err_hook;
2154                 }
2155         }
2156
2157         return 0;
2158
2159 err_hook:
2160         list_for_each_entry_safe(hook, next, hook_list, list) {
2161                 list_del(&hook->list);
2162                 kfree(hook);
2163         }
2164         return err;
2165 }
2166
2167 struct nft_chain_hook {
2168         u32                             num;
2169         s32                             priority;
2170         const struct nft_chain_type     *type;
2171         struct list_head                list;
2172 };
2173
2174 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2175                                   struct list_head *hook_list,
2176                                   struct netlink_ext_ack *extack, u32 flags)
2177 {
2178         struct nft_hook *hook;
2179         int err;
2180
2181         if (tb[NFTA_HOOK_DEV]) {
2182                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2183                 if (IS_ERR(hook)) {
2184                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2185                         return PTR_ERR(hook);
2186                 }
2187
2188                 list_add_tail(&hook->list, hook_list);
2189         } else if (tb[NFTA_HOOK_DEVS]) {
2190                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2191                                                    hook_list, extack);
2192                 if (err < 0)
2193                         return err;
2194
2195         }
2196
2197         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2198             list_empty(hook_list))
2199                 return -EINVAL;
2200
2201         return 0;
2202 }
2203
2204 static int nft_chain_parse_hook(struct net *net,
2205                                 struct nft_base_chain *basechain,
2206                                 const struct nlattr * const nla[],
2207                                 struct nft_chain_hook *hook, u8 family,
2208                                 u32 flags, struct netlink_ext_ack *extack)
2209 {
2210         struct nftables_pernet *nft_net = nft_pernet(net);
2211         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2212         const struct nft_chain_type *type;
2213         int err;
2214
2215         lockdep_assert_held(&nft_net->commit_mutex);
2216         lockdep_nfnl_nft_mutex_not_held();
2217
2218         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2219                                           nla[NFTA_CHAIN_HOOK],
2220                                           nft_hook_policy, NULL);
2221         if (err < 0)
2222                 return err;
2223
2224         if (!basechain) {
2225                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2226                     !ha[NFTA_HOOK_PRIORITY]) {
2227                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2228                         return -ENOENT;
2229                 }
2230
2231                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2232                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2233
2234                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2235                 if (!type)
2236                         return -EOPNOTSUPP;
2237
2238                 if (nla[NFTA_CHAIN_TYPE]) {
2239                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2240                                                            family, true);
2241                         if (IS_ERR(type)) {
2242                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2243                                 return PTR_ERR(type);
2244                         }
2245                 }
2246                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2247                         return -EOPNOTSUPP;
2248
2249                 if (type->type == NFT_CHAIN_T_NAT &&
2250                     hook->priority <= NF_IP_PRI_CONNTRACK)
2251                         return -EOPNOTSUPP;
2252         } else {
2253                 if (ha[NFTA_HOOK_HOOKNUM]) {
2254                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2255                         if (hook->num != basechain->ops.hooknum)
2256                                 return -EOPNOTSUPP;
2257                 }
2258                 if (ha[NFTA_HOOK_PRIORITY]) {
2259                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2260                         if (hook->priority != basechain->ops.priority)
2261                                 return -EOPNOTSUPP;
2262                 }
2263
2264                 if (nla[NFTA_CHAIN_TYPE]) {
2265                         type = __nf_tables_chain_type_lookup(nla[NFTA_CHAIN_TYPE],
2266                                                              family);
2267                         if (!type) {
2268                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2269                                 return -ENOENT;
2270                         }
2271                 } else {
2272                         type = basechain->type;
2273                 }
2274         }
2275
2276         if (!try_module_get(type->owner)) {
2277                 if (nla[NFTA_CHAIN_TYPE])
2278                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2279                 return -ENOENT;
2280         }
2281
2282         hook->type = type;
2283
2284         INIT_LIST_HEAD(&hook->list);
2285         if (nft_base_chain_netdev(family, hook->num)) {
2286                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2287                 if (err < 0) {
2288                         module_put(type->owner);
2289                         return err;
2290                 }
2291         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2292                 module_put(type->owner);
2293                 return -EOPNOTSUPP;
2294         }
2295
2296         return 0;
2297 }
2298
2299 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2300 {
2301         struct nft_hook *h, *next;
2302
2303         list_for_each_entry_safe(h, next, &hook->list, list) {
2304                 list_del(&h->list);
2305                 kfree(h);
2306         }
2307         module_put(hook->type->owner);
2308 }
2309
2310 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2311 {
2312         struct nft_rule_dp_last *lrule;
2313
2314         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2315
2316         lrule = (struct nft_rule_dp_last *)ptr;
2317         lrule->end.is_last = 1;
2318         lrule->chain = chain;
2319         /* blob size does not include the trailer rule */
2320 }
2321
2322 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2323                                                          unsigned int size)
2324 {
2325         struct nft_rule_blob *blob;
2326
2327         if (size > INT_MAX)
2328                 return NULL;
2329
2330         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2331
2332         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2333         if (!blob)
2334                 return NULL;
2335
2336         blob->size = 0;
2337         nft_last_rule(chain, blob->data);
2338
2339         return blob;
2340 }
2341
2342 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2343                                     const struct nft_chain_hook *hook,
2344                                     struct nft_chain *chain)
2345 {
2346         ops->pf                 = family;
2347         ops->hooknum            = hook->num;
2348         ops->priority           = hook->priority;
2349         ops->priv               = chain;
2350         ops->hook               = hook->type->hooks[ops->hooknum];
2351         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2352 }
2353
2354 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2355                               struct nft_chain_hook *hook, u32 flags)
2356 {
2357         struct nft_chain *chain;
2358         struct nft_hook *h;
2359
2360         basechain->type = hook->type;
2361         INIT_LIST_HEAD(&basechain->hook_list);
2362         chain = &basechain->chain;
2363
2364         if (nft_base_chain_netdev(family, hook->num)) {
2365                 list_splice_init(&hook->list, &basechain->hook_list);
2366                 list_for_each_entry(h, &basechain->hook_list, list)
2367                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2368         }
2369         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2370
2371         chain->flags |= NFT_CHAIN_BASE | flags;
2372         basechain->policy = NF_ACCEPT;
2373         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2374             !nft_chain_offload_support(basechain)) {
2375                 list_splice_init(&basechain->hook_list, &hook->list);
2376                 return -EOPNOTSUPP;
2377         }
2378
2379         flow_block_init(&basechain->flow_block);
2380
2381         return 0;
2382 }
2383
2384 int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2385 {
2386         int err;
2387
2388         err = rhltable_insert_key(&table->chains_ht, chain->name,
2389                                   &chain->rhlhead, nft_chain_ht_params);
2390         if (err)
2391                 return err;
2392
2393         list_add_tail_rcu(&chain->list, &table->chains);
2394
2395         return 0;
2396 }
2397
2398 static u64 chain_id;
2399
2400 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2401                               u8 policy, u32 flags,
2402                               struct netlink_ext_ack *extack)
2403 {
2404         const struct nlattr * const *nla = ctx->nla;
2405         struct nft_table *table = ctx->table;
2406         struct nft_base_chain *basechain;
2407         struct net *net = ctx->net;
2408         char name[NFT_NAME_MAXLEN];
2409         struct nft_rule_blob *blob;
2410         struct nft_trans *trans;
2411         struct nft_chain *chain;
2412         int err;
2413
2414         if (nla[NFTA_CHAIN_HOOK]) {
2415                 struct nft_stats __percpu *stats = NULL;
2416                 struct nft_chain_hook hook = {};
2417
2418                 if (flags & NFT_CHAIN_BINDING)
2419                         return -EOPNOTSUPP;
2420
2421                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2422                                            extack);
2423                 if (err < 0)
2424                         return err;
2425
2426                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2427                 if (basechain == NULL) {
2428                         nft_chain_release_hook(&hook);
2429                         return -ENOMEM;
2430                 }
2431                 chain = &basechain->chain;
2432
2433                 if (nla[NFTA_CHAIN_COUNTERS]) {
2434                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2435                         if (IS_ERR(stats)) {
2436                                 nft_chain_release_hook(&hook);
2437                                 kfree(basechain);
2438                                 return PTR_ERR(stats);
2439                         }
2440                         rcu_assign_pointer(basechain->stats, stats);
2441                 }
2442
2443                 err = nft_basechain_init(basechain, family, &hook, flags);
2444                 if (err < 0) {
2445                         nft_chain_release_hook(&hook);
2446                         kfree(basechain);
2447                         free_percpu(stats);
2448                         return err;
2449                 }
2450                 if (stats)
2451                         static_branch_inc(&nft_counters_enabled);
2452         } else {
2453                 if (flags & NFT_CHAIN_BASE)
2454                         return -EINVAL;
2455                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2456                         return -EOPNOTSUPP;
2457
2458                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2459                 if (chain == NULL)
2460                         return -ENOMEM;
2461
2462                 chain->flags = flags;
2463         }
2464         ctx->chain = chain;
2465
2466         INIT_LIST_HEAD(&chain->rules);
2467         chain->handle = nf_tables_alloc_handle(table);
2468         chain->table = table;
2469
2470         if (nla[NFTA_CHAIN_NAME]) {
2471                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2472         } else {
2473                 if (!(flags & NFT_CHAIN_BINDING)) {
2474                         err = -EINVAL;
2475                         goto err_destroy_chain;
2476                 }
2477
2478                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2479                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2480         }
2481
2482         if (!chain->name) {
2483                 err = -ENOMEM;
2484                 goto err_destroy_chain;
2485         }
2486
2487         if (nla[NFTA_CHAIN_USERDATA]) {
2488                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2489                 if (chain->udata == NULL) {
2490                         err = -ENOMEM;
2491                         goto err_destroy_chain;
2492                 }
2493                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2494         }
2495
2496         blob = nf_tables_chain_alloc_rules(chain, 0);
2497         if (!blob) {
2498                 err = -ENOMEM;
2499                 goto err_destroy_chain;
2500         }
2501
2502         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2503         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2504
2505         err = nf_tables_register_hook(net, table, chain);
2506         if (err < 0)
2507                 goto err_destroy_chain;
2508
2509         if (!nft_use_inc(&table->use)) {
2510                 err = -EMFILE;
2511                 goto err_use;
2512         }
2513
2514         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2515         if (IS_ERR(trans)) {
2516                 err = PTR_ERR(trans);
2517                 goto err_unregister_hook;
2518         }
2519
2520         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2521         if (nft_is_base_chain(chain))
2522                 nft_trans_chain_policy(trans) = policy;
2523
2524         err = nft_chain_add(table, chain);
2525         if (err < 0) {
2526                 nft_trans_destroy(trans);
2527                 goto err_unregister_hook;
2528         }
2529
2530         return 0;
2531
2532 err_unregister_hook:
2533         nft_use_dec_restore(&table->use);
2534 err_use:
2535         nf_tables_unregister_hook(net, table, chain);
2536 err_destroy_chain:
2537         nf_tables_chain_destroy(ctx);
2538
2539         return err;
2540 }
2541
2542 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2543                               u32 flags, const struct nlattr *attr,
2544                               struct netlink_ext_ack *extack)
2545 {
2546         const struct nlattr * const *nla = ctx->nla;
2547         struct nft_base_chain *basechain = NULL;
2548         struct nft_table *table = ctx->table;
2549         struct nft_chain *chain = ctx->chain;
2550         struct nft_chain_hook hook = {};
2551         struct nft_stats *stats = NULL;
2552         struct nft_hook *h, *next;
2553         struct nf_hook_ops *ops;
2554         struct nft_trans *trans;
2555         bool unregister = false;
2556         int err;
2557
2558         if (chain->flags ^ flags)
2559                 return -EOPNOTSUPP;
2560
2561         INIT_LIST_HEAD(&hook.list);
2562
2563         if (nla[NFTA_CHAIN_HOOK]) {
2564                 if (!nft_is_base_chain(chain)) {
2565                         NL_SET_BAD_ATTR(extack, attr);
2566                         return -EEXIST;
2567                 }
2568
2569                 basechain = nft_base_chain(chain);
2570                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2571                                            ctx->family, flags, extack);
2572                 if (err < 0)
2573                         return err;
2574
2575                 if (basechain->type != hook.type) {
2576                         nft_chain_release_hook(&hook);
2577                         NL_SET_BAD_ATTR(extack, attr);
2578                         return -EEXIST;
2579                 }
2580
2581                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2582                         list_for_each_entry_safe(h, next, &hook.list, list) {
2583                                 h->ops.pf       = basechain->ops.pf;
2584                                 h->ops.hooknum  = basechain->ops.hooknum;
2585                                 h->ops.priority = basechain->ops.priority;
2586                                 h->ops.priv     = basechain->ops.priv;
2587                                 h->ops.hook     = basechain->ops.hook;
2588
2589                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2590                                         list_del(&h->list);
2591                                         kfree(h);
2592                                 }
2593                         }
2594                 } else {
2595                         ops = &basechain->ops;
2596                         if (ops->hooknum != hook.num ||
2597                             ops->priority != hook.priority) {
2598                                 nft_chain_release_hook(&hook);
2599                                 NL_SET_BAD_ATTR(extack, attr);
2600                                 return -EEXIST;
2601                         }
2602                 }
2603         }
2604
2605         if (nla[NFTA_CHAIN_HANDLE] &&
2606             nla[NFTA_CHAIN_NAME]) {
2607                 struct nft_chain *chain2;
2608
2609                 chain2 = nft_chain_lookup(ctx->net, table,
2610                                           nla[NFTA_CHAIN_NAME], genmask);
2611                 if (!IS_ERR(chain2)) {
2612                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2613                         err = -EEXIST;
2614                         goto err_hooks;
2615                 }
2616         }
2617
2618         if (nla[NFTA_CHAIN_COUNTERS]) {
2619                 if (!nft_is_base_chain(chain)) {
2620                         err = -EOPNOTSUPP;
2621                         goto err_hooks;
2622                 }
2623
2624                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2625                 if (IS_ERR(stats)) {
2626                         err = PTR_ERR(stats);
2627                         goto err_hooks;
2628                 }
2629         }
2630
2631         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2632             nft_is_base_chain(chain) &&
2633             !list_empty(&hook.list)) {
2634                 basechain = nft_base_chain(chain);
2635                 ops = &basechain->ops;
2636
2637                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2638                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2639                         if (err < 0)
2640                                 goto err_hooks;
2641                 }
2642         }
2643
2644         unregister = true;
2645         err = -ENOMEM;
2646         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2647                                 sizeof(struct nft_trans_chain));
2648         if (trans == NULL)
2649                 goto err_trans;
2650
2651         nft_trans_chain_stats(trans) = stats;
2652         nft_trans_chain_update(trans) = true;
2653
2654         if (nla[NFTA_CHAIN_POLICY])
2655                 nft_trans_chain_policy(trans) = policy;
2656         else
2657                 nft_trans_chain_policy(trans) = -1;
2658
2659         if (nla[NFTA_CHAIN_HANDLE] &&
2660             nla[NFTA_CHAIN_NAME]) {
2661                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2662                 struct nft_trans *tmp;
2663                 char *name;
2664
2665                 err = -ENOMEM;
2666                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2667                 if (!name)
2668                         goto err_trans;
2669
2670                 err = -EEXIST;
2671                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2672                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2673                             tmp->ctx.table == table &&
2674                             nft_trans_chain_update(tmp) &&
2675                             nft_trans_chain_name(tmp) &&
2676                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2677                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2678                                 kfree(name);
2679                                 goto err_trans;
2680                         }
2681                 }
2682
2683                 nft_trans_chain_name(trans) = name;
2684         }
2685
2686         nft_trans_basechain(trans) = basechain;
2687         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2688         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2689         if (nla[NFTA_CHAIN_HOOK])
2690                 module_put(hook.type->owner);
2691
2692         nft_trans_commit_list_add_tail(ctx->net, trans);
2693
2694         return 0;
2695
2696 err_trans:
2697         free_percpu(stats);
2698         kfree(trans);
2699 err_hooks:
2700         if (nla[NFTA_CHAIN_HOOK]) {
2701                 list_for_each_entry_safe(h, next, &hook.list, list) {
2702                         if (unregister)
2703                                 nf_unregister_net_hook(ctx->net, &h->ops);
2704                         list_del(&h->list);
2705                         kfree_rcu(h, rcu);
2706                 }
2707                 module_put(hook.type->owner);
2708         }
2709
2710         return err;
2711 }
2712
2713 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2714                                                const struct nft_table *table,
2715                                                const struct nlattr *nla, u8 genmask)
2716 {
2717         struct nftables_pernet *nft_net = nft_pernet(net);
2718         u32 id = ntohl(nla_get_be32(nla));
2719         struct nft_trans *trans;
2720
2721         list_for_each_entry(trans, &nft_net->commit_list, list) {
2722                 struct nft_chain *chain = trans->ctx.chain;
2723
2724                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2725                     chain->table == table &&
2726                     id == nft_trans_chain_id(trans) &&
2727                     nft_active_genmask(chain, genmask))
2728                         return chain;
2729         }
2730         return ERR_PTR(-ENOENT);
2731 }
2732
2733 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2734                               const struct nlattr * const nla[])
2735 {
2736         struct nftables_pernet *nft_net = nft_pernet(info->net);
2737         struct netlink_ext_ack *extack = info->extack;
2738         u8 genmask = nft_genmask_next(info->net);
2739         u8 family = info->nfmsg->nfgen_family;
2740         struct nft_chain *chain = NULL;
2741         struct net *net = info->net;
2742         const struct nlattr *attr;
2743         struct nft_table *table;
2744         u8 policy = NF_ACCEPT;
2745         struct nft_ctx ctx;
2746         u64 handle = 0;
2747         u32 flags = 0;
2748
2749         lockdep_assert_held(&nft_net->commit_mutex);
2750
2751         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2752                                  NETLINK_CB(skb).portid);
2753         if (IS_ERR(table)) {
2754                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2755                 return PTR_ERR(table);
2756         }
2757
2758         chain = NULL;
2759         attr = nla[NFTA_CHAIN_NAME];
2760
2761         if (nla[NFTA_CHAIN_HANDLE]) {
2762                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2763                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2764                 if (IS_ERR(chain)) {
2765                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2766                         return PTR_ERR(chain);
2767                 }
2768                 attr = nla[NFTA_CHAIN_HANDLE];
2769         } else if (nla[NFTA_CHAIN_NAME]) {
2770                 chain = nft_chain_lookup(net, table, attr, genmask);
2771                 if (IS_ERR(chain)) {
2772                         if (PTR_ERR(chain) != -ENOENT) {
2773                                 NL_SET_BAD_ATTR(extack, attr);
2774                                 return PTR_ERR(chain);
2775                         }
2776                         chain = NULL;
2777                 }
2778         } else if (!nla[NFTA_CHAIN_ID]) {
2779                 return -EINVAL;
2780         }
2781
2782         if (nla[NFTA_CHAIN_POLICY]) {
2783                 if (chain != NULL &&
2784                     !nft_is_base_chain(chain)) {
2785                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2786                         return -EOPNOTSUPP;
2787                 }
2788
2789                 if (chain == NULL &&
2790                     nla[NFTA_CHAIN_HOOK] == NULL) {
2791                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2792                         return -EOPNOTSUPP;
2793                 }
2794
2795                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2796                 switch (policy) {
2797                 case NF_DROP:
2798                 case NF_ACCEPT:
2799                         break;
2800                 default:
2801                         return -EINVAL;
2802                 }
2803         }
2804
2805         if (nla[NFTA_CHAIN_FLAGS])
2806                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2807         else if (chain)
2808                 flags = chain->flags;
2809
2810         if (flags & ~NFT_CHAIN_FLAGS)
2811                 return -EOPNOTSUPP;
2812
2813         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2814
2815         if (chain != NULL) {
2816                 if (chain->flags & NFT_CHAIN_BINDING)
2817                         return -EINVAL;
2818
2819                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2820                         NL_SET_BAD_ATTR(extack, attr);
2821                         return -EEXIST;
2822                 }
2823                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2824                         return -EOPNOTSUPP;
2825
2826                 flags |= chain->flags & NFT_CHAIN_BASE;
2827                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2828                                           extack);
2829         }
2830
2831         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2832 }
2833
2834 static int nft_delchain_hook(struct nft_ctx *ctx,
2835                              struct nft_base_chain *basechain,
2836                              struct netlink_ext_ack *extack)
2837 {
2838         const struct nft_chain *chain = &basechain->chain;
2839         const struct nlattr * const *nla = ctx->nla;
2840         struct nft_chain_hook chain_hook = {};
2841         struct nft_hook *this, *hook;
2842         LIST_HEAD(chain_del_list);
2843         struct nft_trans *trans;
2844         int err;
2845
2846         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
2847                                    ctx->family, chain->flags, extack);
2848         if (err < 0)
2849                 return err;
2850
2851         list_for_each_entry(this, &chain_hook.list, list) {
2852                 hook = nft_hook_list_find(&basechain->hook_list, this);
2853                 if (!hook) {
2854                         err = -ENOENT;
2855                         goto err_chain_del_hook;
2856                 }
2857                 list_move(&hook->list, &chain_del_list);
2858         }
2859
2860         trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN,
2861                                 sizeof(struct nft_trans_chain));
2862         if (!trans) {
2863                 err = -ENOMEM;
2864                 goto err_chain_del_hook;
2865         }
2866
2867         nft_trans_basechain(trans) = basechain;
2868         nft_trans_chain_update(trans) = true;
2869         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2870         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
2871         nft_chain_release_hook(&chain_hook);
2872
2873         nft_trans_commit_list_add_tail(ctx->net, trans);
2874
2875         return 0;
2876
2877 err_chain_del_hook:
2878         list_splice(&chain_del_list, &basechain->hook_list);
2879         nft_chain_release_hook(&chain_hook);
2880
2881         return err;
2882 }
2883
2884 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2885                               const struct nlattr * const nla[])
2886 {
2887         struct netlink_ext_ack *extack = info->extack;
2888         u8 genmask = nft_genmask_next(info->net);
2889         u8 family = info->nfmsg->nfgen_family;
2890         struct net *net = info->net;
2891         const struct nlattr *attr;
2892         struct nft_table *table;
2893         struct nft_chain *chain;
2894         struct nft_rule *rule;
2895         struct nft_ctx ctx;
2896         u64 handle;
2897         u32 use;
2898         int err;
2899
2900         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2901                                  NETLINK_CB(skb).portid);
2902         if (IS_ERR(table)) {
2903                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2904                 return PTR_ERR(table);
2905         }
2906
2907         if (nla[NFTA_CHAIN_HANDLE]) {
2908                 attr = nla[NFTA_CHAIN_HANDLE];
2909                 handle = be64_to_cpu(nla_get_be64(attr));
2910                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2911         } else {
2912                 attr = nla[NFTA_CHAIN_NAME];
2913                 chain = nft_chain_lookup(net, table, attr, genmask);
2914         }
2915         if (IS_ERR(chain)) {
2916                 if (PTR_ERR(chain) == -ENOENT &&
2917                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
2918                         return 0;
2919
2920                 NL_SET_BAD_ATTR(extack, attr);
2921                 return PTR_ERR(chain);
2922         }
2923
2924         if (nft_chain_binding(chain))
2925                 return -EOPNOTSUPP;
2926
2927         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2928
2929         if (nla[NFTA_CHAIN_HOOK]) {
2930                 if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
2931                         return -EOPNOTSUPP;
2932
2933                 if (nft_is_base_chain(chain)) {
2934                         struct nft_base_chain *basechain = nft_base_chain(chain);
2935
2936                         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
2937                                 return nft_delchain_hook(&ctx, basechain, extack);
2938                 }
2939         }
2940
2941         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
2942             chain->use > 0)
2943                 return -EBUSY;
2944
2945         use = chain->use;
2946         list_for_each_entry(rule, &chain->rules, list) {
2947                 if (!nft_is_active_next(net, rule))
2948                         continue;
2949                 use--;
2950
2951                 err = nft_delrule(&ctx, rule);
2952                 if (err < 0)
2953                         return err;
2954         }
2955
2956         /* There are rules and elements that are still holding references to us,
2957          * we cannot do a recursive removal in this case.
2958          */
2959         if (use > 0) {
2960                 NL_SET_BAD_ATTR(extack, attr);
2961                 return -EBUSY;
2962         }
2963
2964         return nft_delchain(&ctx);
2965 }
2966
2967 /*
2968  * Expressions
2969  */
2970
2971 /**
2972  *      nft_register_expr - register nf_tables expr type
2973  *      @type: expr type
2974  *
2975  *      Registers the expr type for use with nf_tables. Returns zero on
2976  *      success or a negative errno code otherwise.
2977  */
2978 int nft_register_expr(struct nft_expr_type *type)
2979 {
2980         if (WARN_ON_ONCE(type->maxattr > NFT_EXPR_MAXATTR))
2981                 return -ENOMEM;
2982
2983         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2984         if (type->family == NFPROTO_UNSPEC)
2985                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2986         else
2987                 list_add_rcu(&type->list, &nf_tables_expressions);
2988         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2989         return 0;
2990 }
2991 EXPORT_SYMBOL_GPL(nft_register_expr);
2992
2993 /**
2994  *      nft_unregister_expr - unregister nf_tables expr type
2995  *      @type: expr type
2996  *
2997  *      Unregisters the expr typefor use with nf_tables.
2998  */
2999 void nft_unregister_expr(struct nft_expr_type *type)
3000 {
3001         nfnl_lock(NFNL_SUBSYS_NFTABLES);
3002         list_del_rcu(&type->list);
3003         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
3004 }
3005 EXPORT_SYMBOL_GPL(nft_unregister_expr);
3006
3007 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
3008                                                        struct nlattr *nla)
3009 {
3010         const struct nft_expr_type *type, *candidate = NULL;
3011
3012         list_for_each_entry(type, &nf_tables_expressions, list) {
3013                 if (!nla_strcmp(nla, type->name)) {
3014                         if (!type->family && !candidate)
3015                                 candidate = type;
3016                         else if (type->family == family)
3017                                 candidate = type;
3018                 }
3019         }
3020         return candidate;
3021 }
3022
3023 #ifdef CONFIG_MODULES
3024 static int nft_expr_type_request_module(struct net *net, u8 family,
3025                                         struct nlattr *nla)
3026 {
3027         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
3028                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
3029                 return -EAGAIN;
3030
3031         return 0;
3032 }
3033 #endif
3034
3035 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
3036                                                      u8 family,
3037                                                      struct nlattr *nla)
3038 {
3039         const struct nft_expr_type *type;
3040
3041         if (nla == NULL)
3042                 return ERR_PTR(-EINVAL);
3043
3044         type = __nft_expr_type_get(family, nla);
3045         if (type != NULL && try_module_get(type->owner))
3046                 return type;
3047
3048         lockdep_nfnl_nft_mutex_not_held();
3049 #ifdef CONFIG_MODULES
3050         if (type == NULL) {
3051                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
3052                         return ERR_PTR(-EAGAIN);
3053
3054                 if (nft_request_module(net, "nft-expr-%.*s",
3055                                        nla_len(nla),
3056                                        (char *)nla_data(nla)) == -EAGAIN)
3057                         return ERR_PTR(-EAGAIN);
3058         }
3059 #endif
3060         return ERR_PTR(-ENOENT);
3061 }
3062
3063 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
3064         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
3065                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
3066         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
3067 };
3068
3069 static int nf_tables_fill_expr_info(struct sk_buff *skb,
3070                                     const struct nft_expr *expr, bool reset)
3071 {
3072         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
3073                 goto nla_put_failure;
3074
3075         if (expr->ops->dump) {
3076                 struct nlattr *data = nla_nest_start_noflag(skb,
3077                                                             NFTA_EXPR_DATA);
3078                 if (data == NULL)
3079                         goto nla_put_failure;
3080                 if (expr->ops->dump(skb, expr, reset) < 0)
3081                         goto nla_put_failure;
3082                 nla_nest_end(skb, data);
3083         }
3084
3085         return skb->len;
3086
3087 nla_put_failure:
3088         return -1;
3089 };
3090
3091 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
3092                   const struct nft_expr *expr, bool reset)
3093 {
3094         struct nlattr *nest;
3095
3096         nest = nla_nest_start_noflag(skb, attr);
3097         if (!nest)
3098                 goto nla_put_failure;
3099         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
3100                 goto nla_put_failure;
3101         nla_nest_end(skb, nest);
3102         return 0;
3103
3104 nla_put_failure:
3105         return -1;
3106 }
3107
3108 struct nft_expr_info {
3109         const struct nft_expr_ops       *ops;
3110         const struct nlattr             *attr;
3111         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
3112 };
3113
3114 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
3115                                 const struct nlattr *nla,
3116                                 struct nft_expr_info *info)
3117 {
3118         const struct nft_expr_type *type;
3119         const struct nft_expr_ops *ops;
3120         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3121         int err;
3122
3123         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3124                                           nft_expr_policy, NULL);
3125         if (err < 0)
3126                 return err;
3127
3128         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
3129         if (IS_ERR(type))
3130                 return PTR_ERR(type);
3131
3132         if (tb[NFTA_EXPR_DATA]) {
3133                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3134                                                   tb[NFTA_EXPR_DATA],
3135                                                   type->policy, NULL);
3136                 if (err < 0)
3137                         goto err1;
3138         } else
3139                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
3140
3141         if (type->select_ops != NULL) {
3142                 ops = type->select_ops(ctx,
3143                                        (const struct nlattr * const *)info->tb);
3144                 if (IS_ERR(ops)) {
3145                         err = PTR_ERR(ops);
3146 #ifdef CONFIG_MODULES
3147                         if (err == -EAGAIN)
3148                                 if (nft_expr_type_request_module(ctx->net,
3149                                                                  ctx->family,
3150                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
3151                                         err = -ENOENT;
3152 #endif
3153                         goto err1;
3154                 }
3155         } else
3156                 ops = type->ops;
3157
3158         info->attr = nla;
3159         info->ops = ops;
3160
3161         return 0;
3162
3163 err1:
3164         module_put(type->owner);
3165         return err;
3166 }
3167
3168 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3169                          struct nft_expr_info *info)
3170 {
3171         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3172         const struct nft_expr_type *type;
3173         int err;
3174
3175         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3176                                           nft_expr_policy, NULL);
3177         if (err < 0)
3178                 return err;
3179
3180         if (!tb[NFTA_EXPR_DATA] || !tb[NFTA_EXPR_NAME])
3181                 return -EINVAL;
3182
3183         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3184         if (!type)
3185                 return -ENOENT;
3186
3187         if (!type->inner_ops)
3188                 return -EOPNOTSUPP;
3189
3190         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3191                                           tb[NFTA_EXPR_DATA],
3192                                           type->policy, NULL);
3193         if (err < 0)
3194                 goto err_nla_parse;
3195
3196         info->attr = nla;
3197         info->ops = type->inner_ops;
3198
3199         return 0;
3200
3201 err_nla_parse:
3202         return err;
3203 }
3204
3205 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3206                              const struct nft_expr_info *expr_info,
3207                              struct nft_expr *expr)
3208 {
3209         const struct nft_expr_ops *ops = expr_info->ops;
3210         int err;
3211
3212         expr->ops = ops;
3213         if (ops->init) {
3214                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3215                 if (err < 0)
3216                         goto err1;
3217         }
3218
3219         return 0;
3220 err1:
3221         expr->ops = NULL;
3222         return err;
3223 }
3224
3225 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3226                                    struct nft_expr *expr)
3227 {
3228         const struct nft_expr_type *type = expr->ops->type;
3229
3230         if (expr->ops->destroy)
3231                 expr->ops->destroy(ctx, expr);
3232         module_put(type->owner);
3233 }
3234
3235 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3236                                       const struct nlattr *nla)
3237 {
3238         struct nft_expr_info expr_info;
3239         struct nft_expr *expr;
3240         struct module *owner;
3241         int err;
3242
3243         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3244         if (err < 0)
3245                 goto err_expr_parse;
3246
3247         err = -EOPNOTSUPP;
3248         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3249                 goto err_expr_stateful;
3250
3251         err = -ENOMEM;
3252         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3253         if (expr == NULL)
3254                 goto err_expr_stateful;
3255
3256         err = nf_tables_newexpr(ctx, &expr_info, expr);
3257         if (err < 0)
3258                 goto err_expr_new;
3259
3260         return expr;
3261 err_expr_new:
3262         kfree(expr);
3263 err_expr_stateful:
3264         owner = expr_info.ops->type->owner;
3265         if (expr_info.ops->type->release_ops)
3266                 expr_info.ops->type->release_ops(expr_info.ops);
3267
3268         module_put(owner);
3269 err_expr_parse:
3270         return ERR_PTR(err);
3271 }
3272
3273 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
3274 {
3275         int err;
3276
3277         if (WARN_ON_ONCE(!src->ops->clone))
3278                 return -EINVAL;
3279
3280         dst->ops = src->ops;
3281         err = src->ops->clone(dst, src);
3282         if (err < 0)
3283                 return err;
3284
3285         __module_get(src->ops->type->owner);
3286
3287         return 0;
3288 }
3289
3290 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3291 {
3292         nf_tables_expr_destroy(ctx, expr);
3293         kfree(expr);
3294 }
3295
3296 /*
3297  * Rules
3298  */
3299
3300 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
3301                                           u64 handle)
3302 {
3303         struct nft_rule *rule;
3304
3305         // FIXME: this sucks
3306         list_for_each_entry_rcu(rule, &chain->rules, list) {
3307                 if (handle == rule->handle)
3308                         return rule;
3309         }
3310
3311         return ERR_PTR(-ENOENT);
3312 }
3313
3314 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
3315                                         const struct nlattr *nla)
3316 {
3317         if (nla == NULL)
3318                 return ERR_PTR(-EINVAL);
3319
3320         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
3321 }
3322
3323 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3324         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3325                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3326         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3327                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3328         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3329         [NFTA_RULE_EXPRESSIONS] = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
3330         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3331         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3332         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3333                                     .len = NFT_USERDATA_MAXLEN },
3334         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3335         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3336         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3337 };
3338
3339 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3340                                     u32 portid, u32 seq, int event,
3341                                     u32 flags, int family,
3342                                     const struct nft_table *table,
3343                                     const struct nft_chain *chain,
3344                                     const struct nft_rule *rule, u64 handle,
3345                                     bool reset)
3346 {
3347         struct nlmsghdr *nlh;
3348         const struct nft_expr *expr, *next;
3349         struct nlattr *list;
3350         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3351
3352         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3353                            nft_base_seq(net));
3354         if (!nlh)
3355                 goto nla_put_failure;
3356
3357         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3358                 goto nla_put_failure;
3359         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3360                 goto nla_put_failure;
3361         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3362                          NFTA_RULE_PAD))
3363                 goto nla_put_failure;
3364
3365         if (event != NFT_MSG_DELRULE && handle) {
3366                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3367                                  NFTA_RULE_PAD))
3368                         goto nla_put_failure;
3369         }
3370
3371         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3372                 nft_flow_rule_stats(chain, rule);
3373
3374         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3375         if (list == NULL)
3376                 goto nla_put_failure;
3377         nft_rule_for_each_expr(expr, next, rule) {
3378                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3379                         goto nla_put_failure;
3380         }
3381         nla_nest_end(skb, list);
3382
3383         if (rule->udata) {
3384                 struct nft_userdata *udata = nft_userdata(rule);
3385                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3386                             udata->data) < 0)
3387                         goto nla_put_failure;
3388         }
3389
3390         nlmsg_end(skb, nlh);
3391         return 0;
3392
3393 nla_put_failure:
3394         nlmsg_trim(skb, nlh);
3395         return -1;
3396 }
3397
3398 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3399                                   const struct nft_rule *rule, int event)
3400 {
3401         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3402         const struct nft_rule *prule;
3403         struct sk_buff *skb;
3404         u64 handle = 0;
3405         u16 flags = 0;
3406         int err;
3407
3408         if (!ctx->report &&
3409             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3410                 return;
3411
3412         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3413         if (skb == NULL)
3414                 goto err;
3415
3416         if (event == NFT_MSG_NEWRULE &&
3417             !list_is_first(&rule->list, &ctx->chain->rules) &&
3418             !list_is_last(&rule->list, &ctx->chain->rules)) {
3419                 prule = list_prev_entry(rule, list);
3420                 handle = prule->handle;
3421         }
3422         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3423                 flags |= NLM_F_APPEND;
3424         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3425                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3426
3427         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3428                                        event, flags, ctx->family, ctx->table,
3429                                        ctx->chain, rule, handle, false);
3430         if (err < 0) {
3431                 kfree_skb(skb);
3432                 goto err;
3433         }
3434
3435         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3436         return;
3437 err:
3438         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3439 }
3440
3441 static void audit_log_rule_reset(const struct nft_table *table,
3442                                  unsigned int base_seq,
3443                                  unsigned int nentries)
3444 {
3445         char *buf = kasprintf(GFP_ATOMIC, "%s:%u",
3446                               table->name, base_seq);
3447
3448         audit_log_nfcfg(buf, table->family, nentries,
3449                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3450         kfree(buf);
3451 }
3452
3453 struct nft_rule_dump_ctx {
3454         unsigned int s_idx;
3455         char *table;
3456         char *chain;
3457         bool reset;
3458 };
3459
3460 static int __nf_tables_dump_rules(struct sk_buff *skb,
3461                                   unsigned int *idx,
3462                                   struct netlink_callback *cb,
3463                                   const struct nft_table *table,
3464                                   const struct nft_chain *chain)
3465 {
3466         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3467         struct net *net = sock_net(skb->sk);
3468         const struct nft_rule *rule, *prule;
3469         unsigned int entries = 0;
3470         int ret = 0;
3471         u64 handle;
3472
3473         prule = NULL;
3474         list_for_each_entry_rcu(rule, &chain->rules, list) {
3475                 if (!nft_is_active(net, rule))
3476                         goto cont_skip;
3477                 if (*idx < ctx->s_idx)
3478                         goto cont;
3479                 if (prule)
3480                         handle = prule->handle;
3481                 else
3482                         handle = 0;
3483
3484                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3485                                         cb->nlh->nlmsg_seq,
3486                                         NFT_MSG_NEWRULE,
3487                                         NLM_F_MULTI | NLM_F_APPEND,
3488                                         table->family,
3489                                         table, chain, rule, handle, ctx->reset) < 0) {
3490                         ret = 1;
3491                         break;
3492                 }
3493                 entries++;
3494                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3495 cont:
3496                 prule = rule;
3497 cont_skip:
3498                 (*idx)++;
3499         }
3500
3501         if (ctx->reset && entries)
3502                 audit_log_rule_reset(table, cb->seq, entries);
3503
3504         return ret;
3505 }
3506
3507 static int nf_tables_dump_rules(struct sk_buff *skb,
3508                                 struct netlink_callback *cb)
3509 {
3510         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3511         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3512         struct nft_table *table;
3513         const struct nft_chain *chain;
3514         unsigned int idx = 0;
3515         struct net *net = sock_net(skb->sk);
3516         int family = nfmsg->nfgen_family;
3517         struct nftables_pernet *nft_net;
3518
3519         rcu_read_lock();
3520         nft_net = nft_pernet(net);
3521         cb->seq = READ_ONCE(nft_net->base_seq);
3522
3523         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3524                 if (family != NFPROTO_UNSPEC && family != table->family)
3525                         continue;
3526
3527                 if (ctx->table && strcmp(ctx->table, table->name) != 0)
3528                         continue;
3529
3530                 if (ctx->table && ctx->chain) {
3531                         struct rhlist_head *list, *tmp;
3532
3533                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3534                                                nft_chain_ht_params);
3535                         if (!list)
3536                                 goto done;
3537
3538                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3539                                 if (!nft_is_active(net, chain))
3540                                         continue;
3541                                 __nf_tables_dump_rules(skb, &idx,
3542                                                        cb, table, chain);
3543                                 break;
3544                         }
3545                         goto done;
3546                 }
3547
3548                 list_for_each_entry_rcu(chain, &table->chains, list) {
3549                         if (__nf_tables_dump_rules(skb, &idx,
3550                                                    cb, table, chain))
3551                                 goto done;
3552                 }
3553
3554                 if (ctx->table)
3555                         break;
3556         }
3557 done:
3558         rcu_read_unlock();
3559
3560         ctx->s_idx = idx;
3561         return skb->len;
3562 }
3563
3564 static int nf_tables_dumpreset_rules(struct sk_buff *skb,
3565                                      struct netlink_callback *cb)
3566 {
3567         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
3568         int ret;
3569
3570         /* Mutex is held is to prevent that two concurrent dump-and-reset calls
3571          * do not underrun counters and quotas. The commit_mutex is used for
3572          * the lack a better lock, this is not transaction path.
3573          */
3574         mutex_lock(&nft_net->commit_mutex);
3575         ret = nf_tables_dump_rules(skb, cb);
3576         mutex_unlock(&nft_net->commit_mutex);
3577
3578         return ret;
3579 }
3580
3581 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3582 {
3583         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3584         const struct nlattr * const *nla = cb->data;
3585
3586         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
3587
3588         if (nla[NFTA_RULE_TABLE]) {
3589                 ctx->table = nla_strdup(nla[NFTA_RULE_TABLE], GFP_ATOMIC);
3590                 if (!ctx->table)
3591                         return -ENOMEM;
3592         }
3593         if (nla[NFTA_RULE_CHAIN]) {
3594                 ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN], GFP_ATOMIC);
3595                 if (!ctx->chain) {
3596                         kfree(ctx->table);
3597                         return -ENOMEM;
3598                 }
3599         }
3600         return 0;
3601 }
3602
3603 static int nf_tables_dumpreset_rules_start(struct netlink_callback *cb)
3604 {
3605         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3606
3607         ctx->reset = true;
3608
3609         return nf_tables_dump_rules_start(cb);
3610 }
3611
3612 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3613 {
3614         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3615
3616         kfree(ctx->table);
3617         kfree(ctx->chain);
3618         return 0;
3619 }
3620
3621 /* called with rcu_read_lock held */
3622 static struct sk_buff *
3623 nf_tables_getrule_single(u32 portid, const struct nfnl_info *info,
3624                          const struct nlattr * const nla[], bool reset)
3625 {
3626         struct netlink_ext_ack *extack = info->extack;
3627         u8 genmask = nft_genmask_cur(info->net);
3628         u8 family = info->nfmsg->nfgen_family;
3629         const struct nft_chain *chain;
3630         const struct nft_rule *rule;
3631         struct net *net = info->net;
3632         struct nft_table *table;
3633         struct sk_buff *skb2;
3634         int err;
3635
3636         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3637         if (IS_ERR(table)) {
3638                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3639                 return ERR_CAST(table);
3640         }
3641
3642         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3643         if (IS_ERR(chain)) {
3644                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3645                 return ERR_CAST(chain);
3646         }
3647
3648         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3649         if (IS_ERR(rule)) {
3650                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3651                 return ERR_CAST(rule);
3652         }
3653
3654         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3655         if (!skb2)
3656                 return ERR_PTR(-ENOMEM);
3657
3658         err = nf_tables_fill_rule_info(skb2, net, portid,
3659                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3660                                        family, table, chain, rule, 0, reset);
3661         if (err < 0) {
3662                 kfree_skb(skb2);
3663                 return ERR_PTR(err);
3664         }
3665
3666         return skb2;
3667 }
3668
3669 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3670                              const struct nlattr * const nla[])
3671 {
3672         u32 portid = NETLINK_CB(skb).portid;
3673         struct net *net = info->net;
3674         struct sk_buff *skb2;
3675
3676         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3677                 struct netlink_dump_control c = {
3678                         .start= nf_tables_dump_rules_start,
3679                         .dump = nf_tables_dump_rules,
3680                         .done = nf_tables_dump_rules_done,
3681                         .module = THIS_MODULE,
3682                         .data = (void *)nla,
3683                 };
3684
3685                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3686         }
3687
3688         skb2 = nf_tables_getrule_single(portid, info, nla, false);
3689         if (IS_ERR(skb2))
3690                 return PTR_ERR(skb2);
3691
3692         return nfnetlink_unicast(skb2, net, portid);
3693 }
3694
3695 static int nf_tables_getrule_reset(struct sk_buff *skb,
3696                                    const struct nfnl_info *info,
3697                                    const struct nlattr * const nla[])
3698 {
3699         struct nftables_pernet *nft_net = nft_pernet(info->net);
3700         u32 portid = NETLINK_CB(skb).portid;
3701         struct net *net = info->net;
3702         struct sk_buff *skb2;
3703         char *buf;
3704
3705         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3706                 struct netlink_dump_control c = {
3707                         .start= nf_tables_dumpreset_rules_start,
3708                         .dump = nf_tables_dumpreset_rules,
3709                         .done = nf_tables_dump_rules_done,
3710                         .module = THIS_MODULE,
3711                         .data = (void *)nla,
3712                 };
3713
3714                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3715         }
3716
3717         if (!try_module_get(THIS_MODULE))
3718                 return -EINVAL;
3719         rcu_read_unlock();
3720         mutex_lock(&nft_net->commit_mutex);
3721         skb2 = nf_tables_getrule_single(portid, info, nla, true);
3722         mutex_unlock(&nft_net->commit_mutex);
3723         rcu_read_lock();
3724         module_put(THIS_MODULE);
3725
3726         if (IS_ERR(skb2))
3727                 return PTR_ERR(skb2);
3728
3729         buf = kasprintf(GFP_ATOMIC, "%.*s:%u",
3730                         nla_len(nla[NFTA_RULE_TABLE]),
3731                         (char *)nla_data(nla[NFTA_RULE_TABLE]),
3732                         nft_net->base_seq);
3733         audit_log_nfcfg(buf, info->nfmsg->nfgen_family, 1,
3734                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3735         kfree(buf);
3736
3737         return nfnetlink_unicast(skb2, net, portid);
3738 }
3739
3740 void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
3741 {
3742         struct nft_expr *expr, *next;
3743
3744         /*
3745          * Careful: some expressions might not be initialized in case this
3746          * is called on error from nf_tables_newrule().
3747          */
3748         expr = nft_expr_first(rule);
3749         while (nft_expr_more(rule, expr)) {
3750                 next = nft_expr_next(expr);
3751                 nf_tables_expr_destroy(ctx, expr);
3752                 expr = next;
3753         }
3754         kfree(rule);
3755 }
3756
3757 static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3758 {
3759         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3760         nf_tables_rule_destroy(ctx, rule);
3761 }
3762
3763 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3764 {
3765         struct nft_expr *expr, *last;
3766         const struct nft_data *data;
3767         struct nft_rule *rule;
3768         int err;
3769
3770         if (ctx->level == NFT_JUMP_STACK_SIZE)
3771                 return -EMLINK;
3772
3773         list_for_each_entry(rule, &chain->rules, list) {
3774                 if (fatal_signal_pending(current))
3775                         return -EINTR;
3776
3777                 if (!nft_is_active_next(ctx->net, rule))
3778                         continue;
3779
3780                 nft_rule_for_each_expr(expr, last, rule) {
3781                         if (!expr->ops->validate)
3782                                 continue;
3783
3784                         err = expr->ops->validate(ctx, expr, &data);
3785                         if (err < 0)
3786                                 return err;
3787                 }
3788         }
3789
3790         return 0;
3791 }
3792 EXPORT_SYMBOL_GPL(nft_chain_validate);
3793
3794 static int nft_table_validate(struct net *net, const struct nft_table *table)
3795 {
3796         struct nft_chain *chain;
3797         struct nft_ctx ctx = {
3798                 .net    = net,
3799                 .family = table->family,
3800         };
3801         int err;
3802
3803         list_for_each_entry(chain, &table->chains, list) {
3804                 if (!nft_is_base_chain(chain))
3805                         continue;
3806
3807                 ctx.chain = chain;
3808                 err = nft_chain_validate(&ctx, chain);
3809                 if (err < 0)
3810                         return err;
3811
3812                 cond_resched();
3813         }
3814
3815         return 0;
3816 }
3817
3818 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
3819                          const struct nft_set_iter *iter,
3820                          struct nft_elem_priv *elem_priv)
3821 {
3822         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
3823         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
3824         const struct nft_data *data;
3825         int err;
3826
3827         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
3828             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
3829                 return 0;
3830
3831         data = nft_set_ext_data(ext);
3832         switch (data->verdict.code) {
3833         case NFT_JUMP:
3834         case NFT_GOTO:
3835                 pctx->level++;
3836                 err = nft_chain_validate(ctx, data->verdict.chain);
3837                 if (err < 0)
3838                         return err;
3839                 pctx->level--;
3840                 break;
3841         default:
3842                 break;
3843         }
3844
3845         return 0;
3846 }
3847
3848 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
3849 {
3850         u8 genmask = nft_genmask_next(ctx->net);
3851         struct nft_set_elem_catchall *catchall;
3852         struct nft_set_ext *ext;
3853         int ret = 0;
3854
3855         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
3856                 ext = nft_set_elem_ext(set, catchall->elem);
3857                 if (!nft_set_elem_active(ext, genmask))
3858                         continue;
3859
3860                 ret = nft_setelem_validate(ctx, set, NULL, catchall->elem);
3861                 if (ret < 0)
3862                         return ret;
3863         }
3864
3865         return ret;
3866 }
3867
3868 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3869                                              const struct nft_chain *chain,
3870                                              const struct nlattr *nla);
3871
3872 #define NFT_RULE_MAXEXPRS       128
3873
3874 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3875                              const struct nlattr * const nla[])
3876 {
3877         struct nftables_pernet *nft_net = nft_pernet(info->net);
3878         struct netlink_ext_ack *extack = info->extack;
3879         unsigned int size, i, n, ulen = 0, usize = 0;
3880         u8 genmask = nft_genmask_next(info->net);
3881         struct nft_rule *rule, *old_rule = NULL;
3882         struct nft_expr_info *expr_info = NULL;
3883         u8 family = info->nfmsg->nfgen_family;
3884         struct nft_flow_rule *flow = NULL;
3885         struct net *net = info->net;
3886         struct nft_userdata *udata;
3887         struct nft_table *table;
3888         struct nft_chain *chain;
3889         struct nft_trans *trans;
3890         u64 handle, pos_handle;
3891         struct nft_expr *expr;
3892         struct nft_ctx ctx;
3893         struct nlattr *tmp;
3894         int err, rem;
3895
3896         lockdep_assert_held(&nft_net->commit_mutex);
3897
3898         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3899                                  NETLINK_CB(skb).portid);
3900         if (IS_ERR(table)) {
3901                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3902                 return PTR_ERR(table);
3903         }
3904
3905         if (nla[NFTA_RULE_CHAIN]) {
3906                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3907                                          genmask);
3908                 if (IS_ERR(chain)) {
3909                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3910                         return PTR_ERR(chain);
3911                 }
3912
3913         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3914                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID],
3915                                               genmask);
3916                 if (IS_ERR(chain)) {
3917                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3918                         return PTR_ERR(chain);
3919                 }
3920         } else {
3921                 return -EINVAL;
3922         }
3923
3924         if (nft_chain_is_bound(chain))
3925                 return -EOPNOTSUPP;
3926
3927         if (nla[NFTA_RULE_HANDLE]) {
3928                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3929                 rule = __nft_rule_lookup(chain, handle);
3930                 if (IS_ERR(rule)) {
3931                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3932                         return PTR_ERR(rule);
3933                 }
3934
3935                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3936                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3937                         return -EEXIST;
3938                 }
3939                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3940                         old_rule = rule;
3941                 else
3942                         return -EOPNOTSUPP;
3943         } else {
3944                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
3945                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
3946                         return -EINVAL;
3947                 handle = nf_tables_alloc_handle(table);
3948
3949                 if (nla[NFTA_RULE_POSITION]) {
3950                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3951                         old_rule = __nft_rule_lookup(chain, pos_handle);
3952                         if (IS_ERR(old_rule)) {
3953                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3954                                 return PTR_ERR(old_rule);
3955                         }
3956                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3957                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
3958                         if (IS_ERR(old_rule)) {
3959                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3960                                 return PTR_ERR(old_rule);
3961                         }
3962                 }
3963         }
3964
3965         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3966
3967         n = 0;
3968         size = 0;
3969         if (nla[NFTA_RULE_EXPRESSIONS]) {
3970                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3971                                            sizeof(struct nft_expr_info),
3972                                            GFP_KERNEL);
3973                 if (!expr_info)
3974                         return -ENOMEM;
3975
3976                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3977                         err = -EINVAL;
3978                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3979                                 goto err_release_expr;
3980                         if (n == NFT_RULE_MAXEXPRS)
3981                                 goto err_release_expr;
3982                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
3983                         if (err < 0) {
3984                                 NL_SET_BAD_ATTR(extack, tmp);
3985                                 goto err_release_expr;
3986                         }
3987                         size += expr_info[n].ops->size;
3988                         n++;
3989                 }
3990         }
3991         /* Check for overflow of dlen field */
3992         err = -EFBIG;
3993         if (size >= 1 << 12)
3994                 goto err_release_expr;
3995
3996         if (nla[NFTA_RULE_USERDATA]) {
3997                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3998                 if (ulen > 0)
3999                         usize = sizeof(struct nft_userdata) + ulen;
4000         }
4001
4002         err = -ENOMEM;
4003         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
4004         if (rule == NULL)
4005                 goto err_release_expr;
4006
4007         nft_activate_next(net, rule);
4008
4009         rule->handle = handle;
4010         rule->dlen   = size;
4011         rule->udata  = ulen ? 1 : 0;
4012
4013         if (ulen) {
4014                 udata = nft_userdata(rule);
4015                 udata->len = ulen - 1;
4016                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
4017         }
4018
4019         expr = nft_expr_first(rule);
4020         for (i = 0; i < n; i++) {
4021                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
4022                 if (err < 0) {
4023                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
4024                         goto err_release_rule;
4025                 }
4026
4027                 if (expr_info[i].ops->validate)
4028                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
4029
4030                 expr_info[i].ops = NULL;
4031                 expr = nft_expr_next(expr);
4032         }
4033
4034         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
4035                 flow = nft_flow_rule_create(net, rule);
4036                 if (IS_ERR(flow)) {
4037                         err = PTR_ERR(flow);
4038                         goto err_release_rule;
4039                 }
4040         }
4041
4042         if (!nft_use_inc(&chain->use)) {
4043                 err = -EMFILE;
4044                 goto err_release_rule;
4045         }
4046
4047         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
4048                 if (nft_chain_binding(chain)) {
4049                         err = -EOPNOTSUPP;
4050                         goto err_destroy_flow_rule;
4051                 }
4052
4053                 err = nft_delrule(&ctx, old_rule);
4054                 if (err < 0)
4055                         goto err_destroy_flow_rule;
4056
4057                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4058                 if (trans == NULL) {
4059                         err = -ENOMEM;
4060                         goto err_destroy_flow_rule;
4061                 }
4062                 list_add_tail_rcu(&rule->list, &old_rule->list);
4063         } else {
4064                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4065                 if (!trans) {
4066                         err = -ENOMEM;
4067                         goto err_destroy_flow_rule;
4068                 }
4069
4070                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
4071                         if (old_rule)
4072                                 list_add_rcu(&rule->list, &old_rule->list);
4073                         else
4074                                 list_add_tail_rcu(&rule->list, &chain->rules);
4075                  } else {
4076                         if (old_rule)
4077                                 list_add_tail_rcu(&rule->list, &old_rule->list);
4078                         else
4079                                 list_add_rcu(&rule->list, &chain->rules);
4080                 }
4081         }
4082         kvfree(expr_info);
4083
4084         if (flow)
4085                 nft_trans_flow_rule(trans) = flow;
4086
4087         if (table->validate_state == NFT_VALIDATE_DO)
4088                 return nft_table_validate(net, table);
4089
4090         return 0;
4091
4092 err_destroy_flow_rule:
4093         nft_use_dec_restore(&chain->use);
4094         if (flow)
4095                 nft_flow_rule_destroy(flow);
4096 err_release_rule:
4097         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
4098         nf_tables_rule_destroy(&ctx, rule);
4099 err_release_expr:
4100         for (i = 0; i < n; i++) {
4101                 if (expr_info[i].ops) {
4102                         module_put(expr_info[i].ops->type->owner);
4103                         if (expr_info[i].ops->type->release_ops)
4104                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
4105                 }
4106         }
4107         kvfree(expr_info);
4108
4109         return err;
4110 }
4111
4112 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
4113                                              const struct nft_chain *chain,
4114                                              const struct nlattr *nla)
4115 {
4116         struct nftables_pernet *nft_net = nft_pernet(net);
4117         u32 id = ntohl(nla_get_be32(nla));
4118         struct nft_trans *trans;
4119
4120         list_for_each_entry(trans, &nft_net->commit_list, list) {
4121                 if (trans->msg_type == NFT_MSG_NEWRULE &&
4122                     trans->ctx.chain == chain &&
4123                     id == nft_trans_rule_id(trans))
4124                         return nft_trans_rule(trans);
4125         }
4126         return ERR_PTR(-ENOENT);
4127 }
4128
4129 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
4130                              const struct nlattr * const nla[])
4131 {
4132         struct netlink_ext_ack *extack = info->extack;
4133         u8 genmask = nft_genmask_next(info->net);
4134         u8 family = info->nfmsg->nfgen_family;
4135         struct nft_chain *chain = NULL;
4136         struct net *net = info->net;
4137         struct nft_table *table;
4138         struct nft_rule *rule;
4139         struct nft_ctx ctx;
4140         int err = 0;
4141
4142         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
4143                                  NETLINK_CB(skb).portid);
4144         if (IS_ERR(table)) {
4145                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
4146                 return PTR_ERR(table);
4147         }
4148
4149         if (nla[NFTA_RULE_CHAIN]) {
4150                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
4151                                          genmask);
4152                 if (IS_ERR(chain)) {
4153                         if (PTR_ERR(chain) == -ENOENT &&
4154                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4155                                 return 0;
4156
4157                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
4158                         return PTR_ERR(chain);
4159                 }
4160                 if (nft_chain_binding(chain))
4161                         return -EOPNOTSUPP;
4162         }
4163
4164         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4165
4166         if (chain) {
4167                 if (nla[NFTA_RULE_HANDLE]) {
4168                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
4169                         if (IS_ERR(rule)) {
4170                                 if (PTR_ERR(rule) == -ENOENT &&
4171                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4172                                         return 0;
4173
4174                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4175                                 return PTR_ERR(rule);
4176                         }
4177
4178                         err = nft_delrule(&ctx, rule);
4179                 } else if (nla[NFTA_RULE_ID]) {
4180                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
4181                         if (IS_ERR(rule)) {
4182                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
4183                                 return PTR_ERR(rule);
4184                         }
4185
4186                         err = nft_delrule(&ctx, rule);
4187                 } else {
4188                         err = nft_delrule_by_chain(&ctx);
4189                 }
4190         } else {
4191                 list_for_each_entry(chain, &table->chains, list) {
4192                         if (!nft_is_active_next(net, chain))
4193                                 continue;
4194                         if (nft_chain_binding(chain))
4195                                 continue;
4196
4197                         ctx.chain = chain;
4198                         err = nft_delrule_by_chain(&ctx);
4199                         if (err < 0)
4200                                 break;
4201                 }
4202         }
4203
4204         return err;
4205 }
4206
4207 /*
4208  * Sets
4209  */
4210 static const struct nft_set_type *nft_set_types[] = {
4211         &nft_set_hash_fast_type,
4212         &nft_set_hash_type,
4213         &nft_set_rhash_type,
4214         &nft_set_bitmap_type,
4215         &nft_set_rbtree_type,
4216 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
4217         &nft_set_pipapo_avx2_type,
4218 #endif
4219         &nft_set_pipapo_type,
4220 };
4221
4222 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
4223                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
4224                                  NFT_SET_EVAL)
4225
4226 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
4227 {
4228         return (flags & type->features) == (flags & NFT_SET_FEATURES);
4229 }
4230
4231 /*
4232  * Select a set implementation based on the data characteristics and the
4233  * given policy. The total memory use might not be known if no size is
4234  * given, in that case the amount of memory per element is used.
4235  */
4236 static const struct nft_set_ops *
4237 nft_select_set_ops(const struct nft_ctx *ctx,
4238                    const struct nlattr * const nla[],
4239                    const struct nft_set_desc *desc)
4240 {
4241         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4242         const struct nft_set_ops *ops, *bops;
4243         struct nft_set_estimate est, best;
4244         const struct nft_set_type *type;
4245         u32 flags = 0;
4246         int i;
4247
4248         lockdep_assert_held(&nft_net->commit_mutex);
4249         lockdep_nfnl_nft_mutex_not_held();
4250
4251         if (nla[NFTA_SET_FLAGS] != NULL)
4252                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4253
4254         bops        = NULL;
4255         best.size   = ~0;
4256         best.lookup = ~0;
4257         best.space  = ~0;
4258
4259         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4260                 type = nft_set_types[i];
4261                 ops = &type->ops;
4262
4263                 if (!nft_set_ops_candidate(type, flags))
4264                         continue;
4265                 if (!ops->estimate(desc, flags, &est))
4266                         continue;
4267
4268                 switch (desc->policy) {
4269                 case NFT_SET_POL_PERFORMANCE:
4270                         if (est.lookup < best.lookup)
4271                                 break;
4272                         if (est.lookup == best.lookup &&
4273                             est.space < best.space)
4274                                 break;
4275                         continue;
4276                 case NFT_SET_POL_MEMORY:
4277                         if (!desc->size) {
4278                                 if (est.space < best.space)
4279                                         break;
4280                                 if (est.space == best.space &&
4281                                     est.lookup < best.lookup)
4282                                         break;
4283                         } else if (est.size < best.size || !bops) {
4284                                 break;
4285                         }
4286                         continue;
4287                 default:
4288                         break;
4289                 }
4290
4291                 bops = ops;
4292                 best = est;
4293         }
4294
4295         if (bops != NULL)
4296                 return bops;
4297
4298         return ERR_PTR(-EOPNOTSUPP);
4299 }
4300
4301 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4302         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4303                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4304         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4305                                             .len = NFT_SET_MAXNAMELEN - 1 },
4306         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4307         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4308         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4309         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4310         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4311         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4312         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4313         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4314         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4315         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4316         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4317                                             .len  = NFT_USERDATA_MAXLEN },
4318         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4319         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4320         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4321         [NFTA_SET_EXPRESSIONS]          = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
4322 };
4323
4324 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4325         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4326 };
4327
4328 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4329         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4330         [NFTA_SET_DESC_CONCAT]          = NLA_POLICY_NESTED_ARRAY(nft_concat_policy),
4331 };
4332
4333 static struct nft_set *nft_set_lookup(const struct nft_table *table,
4334                                       const struct nlattr *nla, u8 genmask)
4335 {
4336         struct nft_set *set;
4337
4338         if (nla == NULL)
4339                 return ERR_PTR(-EINVAL);
4340
4341         list_for_each_entry_rcu(set, &table->sets, list) {
4342                 if (!nla_strcmp(nla, set->name) &&
4343                     nft_active_genmask(set, genmask))
4344                         return set;
4345         }
4346         return ERR_PTR(-ENOENT);
4347 }
4348
4349 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4350                                                const struct nlattr *nla,
4351                                                u8 genmask)
4352 {
4353         struct nft_set *set;
4354
4355         list_for_each_entry(set, &table->sets, list) {
4356                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4357                     nft_active_genmask(set, genmask))
4358                         return set;
4359         }
4360         return ERR_PTR(-ENOENT);
4361 }
4362
4363 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4364                                            const struct nft_table *table,
4365                                            const struct nlattr *nla, u8 genmask)
4366 {
4367         struct nftables_pernet *nft_net = nft_pernet(net);
4368         u32 id = ntohl(nla_get_be32(nla));
4369         struct nft_trans *trans;
4370
4371         list_for_each_entry(trans, &nft_net->commit_list, list) {
4372                 if (trans->msg_type == NFT_MSG_NEWSET) {
4373                         struct nft_set *set = nft_trans_set(trans);
4374
4375                         if (id == nft_trans_set_id(trans) &&
4376                             set->table == table &&
4377                             nft_active_genmask(set, genmask))
4378                                 return set;
4379                 }
4380         }
4381         return ERR_PTR(-ENOENT);
4382 }
4383
4384 struct nft_set *nft_set_lookup_global(const struct net *net,
4385                                       const struct nft_table *table,
4386                                       const struct nlattr *nla_set_name,
4387                                       const struct nlattr *nla_set_id,
4388                                       u8 genmask)
4389 {
4390         struct nft_set *set;
4391
4392         set = nft_set_lookup(table, nla_set_name, genmask);
4393         if (IS_ERR(set)) {
4394                 if (!nla_set_id)
4395                         return set;
4396
4397                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4398         }
4399         return set;
4400 }
4401 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4402
4403 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4404                                     const char *name)
4405 {
4406         const struct nft_set *i;
4407         const char *p;
4408         unsigned long *inuse;
4409         unsigned int n = 0, min = 0;
4410
4411         p = strchr(name, '%');
4412         if (p != NULL) {
4413                 if (p[1] != 'd' || strchr(p + 2, '%'))
4414                         return -EINVAL;
4415
4416                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4417                 if (inuse == NULL)
4418                         return -ENOMEM;
4419 cont:
4420                 list_for_each_entry(i, &ctx->table->sets, list) {
4421                         int tmp;
4422
4423                         if (!nft_is_active_next(ctx->net, i))
4424                                 continue;
4425                         if (!sscanf(i->name, name, &tmp))
4426                                 continue;
4427                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4428                                 continue;
4429
4430                         set_bit(tmp - min, inuse);
4431                 }
4432
4433                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4434                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4435                         min += BITS_PER_BYTE * PAGE_SIZE;
4436                         memset(inuse, 0, PAGE_SIZE);
4437                         goto cont;
4438                 }
4439                 free_page((unsigned long)inuse);
4440         }
4441
4442         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4443         if (!set->name)
4444                 return -ENOMEM;
4445
4446         list_for_each_entry(i, &ctx->table->sets, list) {
4447                 if (!nft_is_active_next(ctx->net, i))
4448                         continue;
4449                 if (!strcmp(set->name, i->name)) {
4450                         kfree(set->name);
4451                         set->name = NULL;
4452                         return -ENFILE;
4453                 }
4454         }
4455         return 0;
4456 }
4457
4458 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4459 {
4460         u64 ms = be64_to_cpu(nla_get_be64(nla));
4461         u64 max = (u64)(~((u64)0));
4462
4463         max = div_u64(max, NSEC_PER_MSEC);
4464         if (ms >= max)
4465                 return -ERANGE;
4466
4467         ms *= NSEC_PER_MSEC;
4468         *result = nsecs_to_jiffies64(ms);
4469         return 0;
4470 }
4471
4472 __be64 nf_jiffies64_to_msecs(u64 input)
4473 {
4474         return cpu_to_be64(jiffies64_to_msecs(input));
4475 }
4476
4477 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4478                                      const struct nft_set *set)
4479 {
4480         struct nlattr *concat, *field;
4481         int i;
4482
4483         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4484         if (!concat)
4485                 return -ENOMEM;
4486
4487         for (i = 0; i < set->field_count; i++) {
4488                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4489                 if (!field)
4490                         return -ENOMEM;
4491
4492                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4493                                  htonl(set->field_len[i])))
4494                         return -ENOMEM;
4495
4496                 nla_nest_end(skb, field);
4497         }
4498
4499         nla_nest_end(skb, concat);
4500
4501         return 0;
4502 }
4503
4504 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4505                               const struct nft_set *set, u16 event, u16 flags)
4506 {
4507         u64 timeout = READ_ONCE(set->timeout);
4508         u32 gc_int = READ_ONCE(set->gc_int);
4509         u32 portid = ctx->portid;
4510         struct nlmsghdr *nlh;
4511         struct nlattr *nest;
4512         u32 seq = ctx->seq;
4513         int i;
4514
4515         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4516         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4517                            NFNETLINK_V0, nft_base_seq(ctx->net));
4518         if (!nlh)
4519                 goto nla_put_failure;
4520
4521         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4522                 goto nla_put_failure;
4523         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4524                 goto nla_put_failure;
4525         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4526                          NFTA_SET_PAD))
4527                 goto nla_put_failure;
4528
4529         if (event == NFT_MSG_DELSET) {
4530                 nlmsg_end(skb, nlh);
4531                 return 0;
4532         }
4533
4534         if (set->flags != 0)
4535                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4536                         goto nla_put_failure;
4537
4538         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4539                 goto nla_put_failure;
4540         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4541                 goto nla_put_failure;
4542         if (set->flags & NFT_SET_MAP) {
4543                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4544                         goto nla_put_failure;
4545                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4546                         goto nla_put_failure;
4547         }
4548         if (set->flags & NFT_SET_OBJECT &&
4549             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4550                 goto nla_put_failure;
4551
4552         if (timeout &&
4553             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4554                          nf_jiffies64_to_msecs(timeout),
4555                          NFTA_SET_PAD))
4556                 goto nla_put_failure;
4557         if (gc_int &&
4558             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4559                 goto nla_put_failure;
4560
4561         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4562                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4563                         goto nla_put_failure;
4564         }
4565
4566         if (set->udata &&
4567             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4568                 goto nla_put_failure;
4569
4570         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4571         if (!nest)
4572                 goto nla_put_failure;
4573         if (set->size &&
4574             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4575                 goto nla_put_failure;
4576
4577         if (set->field_count > 1 &&
4578             nf_tables_fill_set_concat(skb, set))
4579                 goto nla_put_failure;
4580
4581         nla_nest_end(skb, nest);
4582
4583         if (set->num_exprs == 1) {
4584                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4585                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4586                         goto nla_put_failure;
4587
4588                 nla_nest_end(skb, nest);
4589         } else if (set->num_exprs > 1) {
4590                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4591                 if (nest == NULL)
4592                         goto nla_put_failure;
4593
4594                 for (i = 0; i < set->num_exprs; i++) {
4595                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4596                                           set->exprs[i], false) < 0)
4597                                 goto nla_put_failure;
4598                 }
4599                 nla_nest_end(skb, nest);
4600         }
4601
4602         nlmsg_end(skb, nlh);
4603         return 0;
4604
4605 nla_put_failure:
4606         nlmsg_trim(skb, nlh);
4607         return -1;
4608 }
4609
4610 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4611                                  const struct nft_set *set, int event,
4612                                  gfp_t gfp_flags)
4613 {
4614         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4615         u32 portid = ctx->portid;
4616         struct sk_buff *skb;
4617         u16 flags = 0;
4618         int err;
4619
4620         if (!ctx->report &&
4621             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4622                 return;
4623
4624         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4625         if (skb == NULL)
4626                 goto err;
4627
4628         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4629                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4630
4631         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4632         if (err < 0) {
4633                 kfree_skb(skb);
4634                 goto err;
4635         }
4636
4637         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4638         return;
4639 err:
4640         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4641 }
4642
4643 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4644 {
4645         const struct nft_set *set;
4646         unsigned int idx, s_idx = cb->args[0];
4647         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4648         struct net *net = sock_net(skb->sk);
4649         struct nft_ctx *ctx = cb->data, ctx_set;
4650         struct nftables_pernet *nft_net;
4651
4652         if (cb->args[1])
4653                 return skb->len;
4654
4655         rcu_read_lock();
4656         nft_net = nft_pernet(net);
4657         cb->seq = READ_ONCE(nft_net->base_seq);
4658
4659         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4660                 if (ctx->family != NFPROTO_UNSPEC &&
4661                     ctx->family != table->family)
4662                         continue;
4663
4664                 if (ctx->table && ctx->table != table)
4665                         continue;
4666
4667                 if (cur_table) {
4668                         if (cur_table != table)
4669                                 continue;
4670
4671                         cur_table = NULL;
4672                 }
4673                 idx = 0;
4674                 list_for_each_entry_rcu(set, &table->sets, list) {
4675                         if (idx < s_idx)
4676                                 goto cont;
4677                         if (!nft_is_active(net, set))
4678                                 goto cont;
4679
4680                         ctx_set = *ctx;
4681                         ctx_set.table = table;
4682                         ctx_set.family = table->family;
4683
4684                         if (nf_tables_fill_set(skb, &ctx_set, set,
4685                                                NFT_MSG_NEWSET,
4686                                                NLM_F_MULTI) < 0) {
4687                                 cb->args[0] = idx;
4688                                 cb->args[2] = (unsigned long) table;
4689                                 goto done;
4690                         }
4691                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4692 cont:
4693                         idx++;
4694                 }
4695                 if (s_idx)
4696                         s_idx = 0;
4697         }
4698         cb->args[1] = 1;
4699 done:
4700         rcu_read_unlock();
4701         return skb->len;
4702 }
4703
4704 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4705 {
4706         struct nft_ctx *ctx_dump = NULL;
4707
4708         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4709         if (ctx_dump == NULL)
4710                 return -ENOMEM;
4711
4712         cb->data = ctx_dump;
4713         return 0;
4714 }
4715
4716 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4717 {
4718         kfree(cb->data);
4719         return 0;
4720 }
4721
4722 /* called with rcu_read_lock held */
4723 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4724                             const struct nlattr * const nla[])
4725 {
4726         struct netlink_ext_ack *extack = info->extack;
4727         u8 genmask = nft_genmask_cur(info->net);
4728         u8 family = info->nfmsg->nfgen_family;
4729         struct nft_table *table = NULL;
4730         struct net *net = info->net;
4731         const struct nft_set *set;
4732         struct sk_buff *skb2;
4733         struct nft_ctx ctx;
4734         int err;
4735
4736         if (nla[NFTA_SET_TABLE]) {
4737                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4738                                          genmask, 0);
4739                 if (IS_ERR(table)) {
4740                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4741                         return PTR_ERR(table);
4742                 }
4743         }
4744
4745         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4746
4747         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4748                 struct netlink_dump_control c = {
4749                         .start = nf_tables_dump_sets_start,
4750                         .dump = nf_tables_dump_sets,
4751                         .done = nf_tables_dump_sets_done,
4752                         .data = &ctx,
4753                         .module = THIS_MODULE,
4754                 };
4755
4756                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4757         }
4758
4759         /* Only accept unspec with dump */
4760         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4761                 return -EAFNOSUPPORT;
4762         if (!nla[NFTA_SET_TABLE])
4763                 return -EINVAL;
4764
4765         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4766         if (IS_ERR(set)) {
4767                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4768                 return PTR_ERR(set);
4769         }
4770
4771         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4772         if (skb2 == NULL)
4773                 return -ENOMEM;
4774
4775         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4776         if (err < 0)
4777                 goto err_fill_set_info;
4778
4779         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4780
4781 err_fill_set_info:
4782         kfree_skb(skb2);
4783         return err;
4784 }
4785
4786 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4787                                      struct nft_set_desc *desc)
4788 {
4789         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4790         u32 len;
4791         int err;
4792
4793         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4794                 return -E2BIG;
4795
4796         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4797                                           nft_concat_policy, NULL);
4798         if (err < 0)
4799                 return err;
4800
4801         if (!tb[NFTA_SET_FIELD_LEN])
4802                 return -EINVAL;
4803
4804         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4805         if (!len || len > U8_MAX)
4806                 return -EINVAL;
4807
4808         desc->field_len[desc->field_count++] = len;
4809
4810         return 0;
4811 }
4812
4813 static int nft_set_desc_concat(struct nft_set_desc *desc,
4814                                const struct nlattr *nla)
4815 {
4816         u32 num_regs = 0, key_num_regs = 0;
4817         struct nlattr *attr;
4818         int rem, err, i;
4819
4820         nla_for_each_nested(attr, nla, rem) {
4821                 if (nla_type(attr) != NFTA_LIST_ELEM)
4822                         return -EINVAL;
4823
4824                 err = nft_set_desc_concat_parse(attr, desc);
4825                 if (err < 0)
4826                         return err;
4827         }
4828
4829         for (i = 0; i < desc->field_count; i++)
4830                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4831
4832         key_num_regs = DIV_ROUND_UP(desc->klen, sizeof(u32));
4833         if (key_num_regs != num_regs)
4834                 return -EINVAL;
4835
4836         if (num_regs > NFT_REG32_COUNT)
4837                 return -E2BIG;
4838
4839         return 0;
4840 }
4841
4842 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4843                                     const struct nlattr *nla)
4844 {
4845         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4846         int err;
4847
4848         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4849                                           nft_set_desc_policy, NULL);
4850         if (err < 0)
4851                 return err;
4852
4853         if (da[NFTA_SET_DESC_SIZE] != NULL)
4854                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4855         if (da[NFTA_SET_DESC_CONCAT])
4856                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4857
4858         return err;
4859 }
4860
4861 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4862                               const struct nlattr * const *nla,
4863                               struct nft_expr **exprs, int *num_exprs,
4864                               u32 flags)
4865 {
4866         struct nft_expr *expr;
4867         int err, i;
4868
4869         if (nla[NFTA_SET_EXPR]) {
4870                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4871                 if (IS_ERR(expr)) {
4872                         err = PTR_ERR(expr);
4873                         goto err_set_expr_alloc;
4874                 }
4875                 exprs[0] = expr;
4876                 (*num_exprs)++;
4877         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4878                 struct nlattr *tmp;
4879                 int left;
4880
4881                 if (!(flags & NFT_SET_EXPR)) {
4882                         err = -EINVAL;
4883                         goto err_set_expr_alloc;
4884                 }
4885                 i = 0;
4886                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4887                         if (i == NFT_SET_EXPR_MAX) {
4888                                 err = -E2BIG;
4889                                 goto err_set_expr_alloc;
4890                         }
4891                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4892                                 err = -EINVAL;
4893                                 goto err_set_expr_alloc;
4894                         }
4895                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4896                         if (IS_ERR(expr)) {
4897                                 err = PTR_ERR(expr);
4898                                 goto err_set_expr_alloc;
4899                         }
4900                         exprs[i++] = expr;
4901                         (*num_exprs)++;
4902                 }
4903         }
4904
4905         return 0;
4906
4907 err_set_expr_alloc:
4908         for (i = 0; i < *num_exprs; i++)
4909                 nft_expr_destroy(ctx, exprs[i]);
4910
4911         return err;
4912 }
4913
4914 static bool nft_set_is_same(const struct nft_set *set,
4915                             const struct nft_set_desc *desc,
4916                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4917 {
4918         int i;
4919
4920         if (set->ktype != desc->ktype ||
4921             set->dtype != desc->dtype ||
4922             set->flags != flags ||
4923             set->klen != desc->klen ||
4924             set->dlen != desc->dlen ||
4925             set->field_count != desc->field_count ||
4926             set->num_exprs != num_exprs)
4927                 return false;
4928
4929         for (i = 0; i < desc->field_count; i++) {
4930                 if (set->field_len[i] != desc->field_len[i])
4931                         return false;
4932         }
4933
4934         for (i = 0; i < num_exprs; i++) {
4935                 if (set->exprs[i]->ops != exprs[i]->ops)
4936                         return false;
4937         }
4938
4939         return true;
4940 }
4941
4942 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
4943                             const struct nlattr * const nla[])
4944 {
4945         struct netlink_ext_ack *extack = info->extack;
4946         u8 genmask = nft_genmask_next(info->net);
4947         u8 family = info->nfmsg->nfgen_family;
4948         const struct nft_set_ops *ops;
4949         struct net *net = info->net;
4950         struct nft_set_desc desc;
4951         struct nft_table *table;
4952         unsigned char *udata;
4953         struct nft_set *set;
4954         struct nft_ctx ctx;
4955         size_t alloc_size;
4956         int num_exprs = 0;
4957         char *name;
4958         int err, i;
4959         u16 udlen;
4960         u32 flags;
4961         u64 size;
4962
4963         if (nla[NFTA_SET_TABLE] == NULL ||
4964             nla[NFTA_SET_NAME] == NULL ||
4965             nla[NFTA_SET_KEY_LEN] == NULL ||
4966             nla[NFTA_SET_ID] == NULL)
4967                 return -EINVAL;
4968
4969         memset(&desc, 0, sizeof(desc));
4970
4971         desc.ktype = NFT_DATA_VALUE;
4972         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4973                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4974                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4975                         return -EINVAL;
4976         }
4977
4978         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4979         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4980                 return -EINVAL;
4981
4982         flags = 0;
4983         if (nla[NFTA_SET_FLAGS] != NULL) {
4984                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4985                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4986                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4987                               NFT_SET_MAP | NFT_SET_EVAL |
4988                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
4989                         return -EOPNOTSUPP;
4990                 /* Only one of these operations is supported */
4991                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4992                              (NFT_SET_MAP | NFT_SET_OBJECT))
4993                         return -EOPNOTSUPP;
4994                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4995                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4996                         return -EOPNOTSUPP;
4997         }
4998
4999         desc.dtype = 0;
5000         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
5001                 if (!(flags & NFT_SET_MAP))
5002                         return -EINVAL;
5003
5004                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
5005                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
5006                     desc.dtype != NFT_DATA_VERDICT)
5007                         return -EINVAL;
5008
5009                 if (desc.dtype != NFT_DATA_VERDICT) {
5010                         if (nla[NFTA_SET_DATA_LEN] == NULL)
5011                                 return -EINVAL;
5012                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
5013                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
5014                                 return -EINVAL;
5015                 } else
5016                         desc.dlen = sizeof(struct nft_verdict);
5017         } else if (flags & NFT_SET_MAP)
5018                 return -EINVAL;
5019
5020         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
5021                 if (!(flags & NFT_SET_OBJECT))
5022                         return -EINVAL;
5023
5024                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
5025                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
5026                     desc.objtype > NFT_OBJECT_MAX)
5027                         return -EOPNOTSUPP;
5028         } else if (flags & NFT_SET_OBJECT)
5029                 return -EINVAL;
5030         else
5031                 desc.objtype = NFT_OBJECT_UNSPEC;
5032
5033         desc.timeout = 0;
5034         if (nla[NFTA_SET_TIMEOUT] != NULL) {
5035                 if (!(flags & NFT_SET_TIMEOUT))
5036                         return -EINVAL;
5037
5038                 if (flags & NFT_SET_ANONYMOUS)
5039                         return -EOPNOTSUPP;
5040
5041                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
5042                 if (err)
5043                         return err;
5044         }
5045         desc.gc_int = 0;
5046         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
5047                 if (!(flags & NFT_SET_TIMEOUT))
5048                         return -EINVAL;
5049
5050                 if (flags & NFT_SET_ANONYMOUS)
5051                         return -EOPNOTSUPP;
5052
5053                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
5054         }
5055
5056         desc.policy = NFT_SET_POL_PERFORMANCE;
5057         if (nla[NFTA_SET_POLICY] != NULL) {
5058                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
5059                 switch (desc.policy) {
5060                 case NFT_SET_POL_PERFORMANCE:
5061                 case NFT_SET_POL_MEMORY:
5062                         break;
5063                 default:
5064                         return -EOPNOTSUPP;
5065                 }
5066         }
5067
5068         if (nla[NFTA_SET_DESC] != NULL) {
5069                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
5070                 if (err < 0)
5071                         return err;
5072
5073                 if (desc.field_count > 1) {
5074                         if (!(flags & NFT_SET_CONCAT))
5075                                 return -EINVAL;
5076                 } else if (flags & NFT_SET_CONCAT) {
5077                         return -EINVAL;
5078                 }
5079         } else if (flags & NFT_SET_CONCAT) {
5080                 return -EINVAL;
5081         }
5082
5083         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
5084                 desc.expr = true;
5085
5086         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
5087                                  NETLINK_CB(skb).portid);
5088         if (IS_ERR(table)) {
5089                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5090                 return PTR_ERR(table);
5091         }
5092
5093         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5094
5095         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
5096         if (IS_ERR(set)) {
5097                 if (PTR_ERR(set) != -ENOENT) {
5098                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5099                         return PTR_ERR(set);
5100                 }
5101         } else {
5102                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
5103
5104                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
5105                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5106                         return -EEXIST;
5107                 }
5108                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
5109                         return -EOPNOTSUPP;
5110
5111                 if (nft_set_is_anonymous(set))
5112                         return -EOPNOTSUPP;
5113
5114                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
5115                 if (err < 0)
5116                         return err;
5117
5118                 err = 0;
5119                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
5120                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5121                         err = -EEXIST;
5122                 }
5123
5124                 for (i = 0; i < num_exprs; i++)
5125                         nft_expr_destroy(&ctx, exprs[i]);
5126
5127                 if (err < 0)
5128                         return err;
5129
5130                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
5131         }
5132
5133         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
5134                 return -ENOENT;
5135
5136         ops = nft_select_set_ops(&ctx, nla, &desc);
5137         if (IS_ERR(ops))
5138                 return PTR_ERR(ops);
5139
5140         udlen = 0;
5141         if (nla[NFTA_SET_USERDATA])
5142                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
5143
5144         size = 0;
5145         if (ops->privsize != NULL)
5146                 size = ops->privsize(nla, &desc);
5147         alloc_size = sizeof(*set) + size + udlen;
5148         if (alloc_size < size || alloc_size > INT_MAX)
5149                 return -ENOMEM;
5150
5151         if (!nft_use_inc(&table->use))
5152                 return -EMFILE;
5153
5154         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
5155         if (!set) {
5156                 err = -ENOMEM;
5157                 goto err_alloc;
5158         }
5159
5160         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
5161         if (!name) {
5162                 err = -ENOMEM;
5163                 goto err_set_name;
5164         }
5165
5166         err = nf_tables_set_alloc_name(&ctx, set, name);
5167         kfree(name);
5168         if (err < 0)
5169                 goto err_set_name;
5170
5171         udata = NULL;
5172         if (udlen) {
5173                 udata = set->data + size;
5174                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
5175         }
5176
5177         INIT_LIST_HEAD(&set->bindings);
5178         INIT_LIST_HEAD(&set->catchall_list);
5179         refcount_set(&set->refs, 1);
5180         set->table = table;
5181         write_pnet(&set->net, net);
5182         set->ops = ops;
5183         set->ktype = desc.ktype;
5184         set->klen = desc.klen;
5185         set->dtype = desc.dtype;
5186         set->objtype = desc.objtype;
5187         set->dlen = desc.dlen;
5188         set->flags = flags;
5189         set->size = desc.size;
5190         set->policy = desc.policy;
5191         set->udlen = udlen;
5192         set->udata = udata;
5193         set->timeout = desc.timeout;
5194         set->gc_int = desc.gc_int;
5195
5196         set->field_count = desc.field_count;
5197         for (i = 0; i < desc.field_count; i++)
5198                 set->field_len[i] = desc.field_len[i];
5199
5200         err = ops->init(set, &desc, nla);
5201         if (err < 0)
5202                 goto err_set_init;
5203
5204         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
5205         if (err < 0)
5206                 goto err_set_destroy;
5207
5208         set->num_exprs = num_exprs;
5209         set->handle = nf_tables_alloc_handle(table);
5210         INIT_LIST_HEAD(&set->pending_update);
5211
5212         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
5213         if (err < 0)
5214                 goto err_set_expr_alloc;
5215
5216         list_add_tail_rcu(&set->list, &table->sets);
5217
5218         return 0;
5219
5220 err_set_expr_alloc:
5221         for (i = 0; i < set->num_exprs; i++)
5222                 nft_expr_destroy(&ctx, set->exprs[i]);
5223 err_set_destroy:
5224         ops->destroy(&ctx, set);
5225 err_set_init:
5226         kfree(set->name);
5227 err_set_name:
5228         kvfree(set);
5229 err_alloc:
5230         nft_use_dec_restore(&table->use);
5231
5232         return err;
5233 }
5234
5235 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
5236                                      struct nft_set *set)
5237 {
5238         struct nft_set_elem_catchall *next, *catchall;
5239
5240         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5241                 list_del_rcu(&catchall->list);
5242                 nf_tables_set_elem_destroy(ctx, set, catchall->elem);
5243                 kfree_rcu(catchall, rcu);
5244         }
5245 }
5246
5247 static void nft_set_put(struct nft_set *set)
5248 {
5249         if (refcount_dec_and_test(&set->refs)) {
5250                 kfree(set->name);
5251                 kvfree(set);
5252         }
5253 }
5254
5255 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5256 {
5257         int i;
5258
5259         if (WARN_ON(set->use > 0))
5260                 return;
5261
5262         for (i = 0; i < set->num_exprs; i++)
5263                 nft_expr_destroy(ctx, set->exprs[i]);
5264
5265         set->ops->destroy(ctx, set);
5266         nft_set_catchall_destroy(ctx, set);
5267         nft_set_put(set);
5268 }
5269
5270 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
5271                             const struct nlattr * const nla[])
5272 {
5273         struct netlink_ext_ack *extack = info->extack;
5274         u8 genmask = nft_genmask_next(info->net);
5275         u8 family = info->nfmsg->nfgen_family;
5276         struct net *net = info->net;
5277         const struct nlattr *attr;
5278         struct nft_table *table;
5279         struct nft_set *set;
5280         struct nft_ctx ctx;
5281
5282         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5283                 return -EAFNOSUPPORT;
5284
5285         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
5286                                  genmask, NETLINK_CB(skb).portid);
5287         if (IS_ERR(table)) {
5288                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5289                 return PTR_ERR(table);
5290         }
5291
5292         if (nla[NFTA_SET_HANDLE]) {
5293                 attr = nla[NFTA_SET_HANDLE];
5294                 set = nft_set_lookup_byhandle(table, attr, genmask);
5295         } else {
5296                 attr = nla[NFTA_SET_NAME];
5297                 set = nft_set_lookup(table, attr, genmask);
5298         }
5299
5300         if (IS_ERR(set)) {
5301                 if (PTR_ERR(set) == -ENOENT &&
5302                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5303                         return 0;
5304
5305                 NL_SET_BAD_ATTR(extack, attr);
5306                 return PTR_ERR(set);
5307         }
5308         if (set->use ||
5309             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5310              atomic_read(&set->nelems) > 0)) {
5311                 NL_SET_BAD_ATTR(extack, attr);
5312                 return -EBUSY;
5313         }
5314
5315         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5316
5317         return nft_delset(&ctx, set);
5318 }
5319
5320 static int nft_validate_register_store(const struct nft_ctx *ctx,
5321                                        enum nft_registers reg,
5322                                        const struct nft_data *data,
5323                                        enum nft_data_types type,
5324                                        unsigned int len);
5325
5326 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5327                                      struct nft_set *set,
5328                                      struct nft_elem_priv *elem_priv)
5329 {
5330         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5331         enum nft_registers dreg;
5332
5333         dreg = nft_type_to_reg(set->dtype);
5334         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5335                                            set->dtype == NFT_DATA_VERDICT ?
5336                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5337                                            set->dlen);
5338 }
5339
5340 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5341                                         struct nft_set *set,
5342                                         const struct nft_set_iter *iter,
5343                                         struct nft_elem_priv *elem_priv)
5344 {
5345         return nft_setelem_data_validate(ctx, set, elem_priv);
5346 }
5347
5348 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5349                                        struct nft_set *set)
5350 {
5351         u8 genmask = nft_genmask_next(ctx->net);
5352         struct nft_set_elem_catchall *catchall;
5353         struct nft_set_ext *ext;
5354         int ret = 0;
5355
5356         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5357                 ext = nft_set_elem_ext(set, catchall->elem);
5358                 if (!nft_set_elem_active(ext, genmask))
5359                         continue;
5360
5361                 ret = nft_setelem_data_validate(ctx, set, catchall->elem);
5362                 if (ret < 0)
5363                         break;
5364         }
5365
5366         return ret;
5367 }
5368
5369 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5370                        struct nft_set_binding *binding)
5371 {
5372         struct nft_set_binding *i;
5373         struct nft_set_iter iter;
5374
5375         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5376                 return -EBUSY;
5377
5378         if (binding->flags & NFT_SET_MAP) {
5379                 /* If the set is already bound to the same chain all
5380                  * jumps are already validated for that chain.
5381                  */
5382                 list_for_each_entry(i, &set->bindings, list) {
5383                         if (i->flags & NFT_SET_MAP &&
5384                             i->chain == binding->chain)
5385                                 goto bind;
5386                 }
5387
5388                 iter.genmask    = nft_genmask_next(ctx->net);
5389                 iter.skip       = 0;
5390                 iter.count      = 0;
5391                 iter.err        = 0;
5392                 iter.fn         = nf_tables_bind_check_setelem;
5393
5394                 set->ops->walk(ctx, set, &iter);
5395                 if (!iter.err)
5396                         iter.err = nft_set_catchall_bind_check(ctx, set);
5397
5398                 if (iter.err < 0)
5399                         return iter.err;
5400         }
5401 bind:
5402         if (!nft_use_inc(&set->use))
5403                 return -EMFILE;
5404
5405         binding->chain = ctx->chain;
5406         list_add_tail_rcu(&binding->list, &set->bindings);
5407         nft_set_trans_bind(ctx, set);
5408
5409         return 0;
5410 }
5411 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5412
5413 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5414                                  struct nft_set_binding *binding, bool event)
5415 {
5416         list_del_rcu(&binding->list);
5417
5418         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5419                 list_del_rcu(&set->list);
5420                 if (event)
5421                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5422                                              GFP_KERNEL);
5423         }
5424 }
5425
5426 static void nft_setelem_data_activate(const struct net *net,
5427                                       const struct nft_set *set,
5428                                       struct nft_elem_priv *elem_priv);
5429
5430 static int nft_mapelem_activate(const struct nft_ctx *ctx,
5431                                 struct nft_set *set,
5432                                 const struct nft_set_iter *iter,
5433                                 struct nft_elem_priv *elem_priv)
5434 {
5435         nft_setelem_data_activate(ctx->net, set, elem_priv);
5436
5437         return 0;
5438 }
5439
5440 static void nft_map_catchall_activate(const struct nft_ctx *ctx,
5441                                       struct nft_set *set)
5442 {
5443         u8 genmask = nft_genmask_next(ctx->net);
5444         struct nft_set_elem_catchall *catchall;
5445         struct nft_set_ext *ext;
5446
5447         list_for_each_entry(catchall, &set->catchall_list, list) {
5448                 ext = nft_set_elem_ext(set, catchall->elem);
5449                 if (!nft_set_elem_active(ext, genmask))
5450                         continue;
5451
5452                 nft_setelem_data_activate(ctx->net, set, catchall->elem);
5453                 break;
5454         }
5455 }
5456
5457 static void nft_map_activate(const struct nft_ctx *ctx, struct nft_set *set)
5458 {
5459         struct nft_set_iter iter = {
5460                 .genmask        = nft_genmask_next(ctx->net),
5461                 .fn             = nft_mapelem_activate,
5462         };
5463
5464         set->ops->walk(ctx, set, &iter);
5465         WARN_ON_ONCE(iter.err);
5466
5467         nft_map_catchall_activate(ctx, set);
5468 }
5469
5470 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5471 {
5472         if (nft_set_is_anonymous(set)) {
5473                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5474                         nft_map_activate(ctx, set);
5475
5476                 nft_clear(ctx->net, set);
5477         }
5478
5479         nft_use_inc_restore(&set->use);
5480 }
5481 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5482
5483 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5484                               struct nft_set_binding *binding,
5485                               enum nft_trans_phase phase)
5486 {
5487         switch (phase) {
5488         case NFT_TRANS_PREPARE_ERROR:
5489                 nft_set_trans_unbind(ctx, set);
5490                 if (nft_set_is_anonymous(set))
5491                         nft_deactivate_next(ctx->net, set);
5492                 else
5493                         list_del_rcu(&binding->list);
5494
5495                 nft_use_dec(&set->use);
5496                 break;
5497         case NFT_TRANS_PREPARE:
5498                 if (nft_set_is_anonymous(set)) {
5499                         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5500                                 nft_map_deactivate(ctx, set);
5501
5502                         nft_deactivate_next(ctx->net, set);
5503                 }
5504                 nft_use_dec(&set->use);
5505                 return;
5506         case NFT_TRANS_ABORT:
5507         case NFT_TRANS_RELEASE:
5508                 if (nft_set_is_anonymous(set) &&
5509                     set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5510                         nft_map_deactivate(ctx, set);
5511
5512                 nft_use_dec(&set->use);
5513                 fallthrough;
5514         default:
5515                 nf_tables_unbind_set(ctx, set, binding,
5516                                      phase == NFT_TRANS_COMMIT);
5517         }
5518 }
5519 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5520
5521 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5522 {
5523         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5524                 nft_set_destroy(ctx, set);
5525 }
5526 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5527
5528 const struct nft_set_ext_type nft_set_ext_types[] = {
5529         [NFT_SET_EXT_KEY]               = {
5530                 .align  = __alignof__(u32),
5531         },
5532         [NFT_SET_EXT_DATA]              = {
5533                 .align  = __alignof__(u32),
5534         },
5535         [NFT_SET_EXT_EXPRESSIONS]       = {
5536                 .align  = __alignof__(struct nft_set_elem_expr),
5537         },
5538         [NFT_SET_EXT_OBJREF]            = {
5539                 .len    = sizeof(struct nft_object *),
5540                 .align  = __alignof__(struct nft_object *),
5541         },
5542         [NFT_SET_EXT_FLAGS]             = {
5543                 .len    = sizeof(u8),
5544                 .align  = __alignof__(u8),
5545         },
5546         [NFT_SET_EXT_TIMEOUT]           = {
5547                 .len    = sizeof(u64),
5548                 .align  = __alignof__(u64),
5549         },
5550         [NFT_SET_EXT_EXPIRATION]        = {
5551                 .len    = sizeof(u64),
5552                 .align  = __alignof__(u64),
5553         },
5554         [NFT_SET_EXT_USERDATA]          = {
5555                 .len    = sizeof(struct nft_userdata),
5556                 .align  = __alignof__(struct nft_userdata),
5557         },
5558         [NFT_SET_EXT_KEY_END]           = {
5559                 .align  = __alignof__(u32),
5560         },
5561 };
5562
5563 /*
5564  * Set elements
5565  */
5566
5567 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5568         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5569         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5570         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5571         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5572         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5573         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5574                                             .len = NFT_USERDATA_MAXLEN },
5575         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5576         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5577                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5578         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5579         [NFTA_SET_ELEM_EXPRESSIONS]     = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
5580 };
5581
5582 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5583         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5584                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5585         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5586                                             .len = NFT_SET_MAXNAMELEN - 1 },
5587         [NFTA_SET_ELEM_LIST_ELEMENTS]   = NLA_POLICY_NESTED_ARRAY(nft_set_elem_policy),
5588         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5589 };
5590
5591 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5592                                   const struct nft_set *set,
5593                                   const struct nft_set_ext *ext,
5594                                   bool reset)
5595 {
5596         struct nft_set_elem_expr *elem_expr;
5597         u32 size, num_exprs = 0;
5598         struct nft_expr *expr;
5599         struct nlattr *nest;
5600
5601         elem_expr = nft_set_ext_expr(ext);
5602         nft_setelem_expr_foreach(expr, elem_expr, size)
5603                 num_exprs++;
5604
5605         if (num_exprs == 1) {
5606                 expr = nft_setelem_expr_at(elem_expr, 0);
5607                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, reset) < 0)
5608                         return -1;
5609
5610                 return 0;
5611         } else if (num_exprs > 1) {
5612                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5613                 if (nest == NULL)
5614                         goto nla_put_failure;
5615
5616                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5617                         expr = nft_setelem_expr_at(elem_expr, size);
5618                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
5619                                 goto nla_put_failure;
5620                 }
5621                 nla_nest_end(skb, nest);
5622         }
5623         return 0;
5624
5625 nla_put_failure:
5626         return -1;
5627 }
5628
5629 static int nf_tables_fill_setelem(struct sk_buff *skb,
5630                                   const struct nft_set *set,
5631                                   const struct nft_elem_priv *elem_priv,
5632                                   bool reset)
5633 {
5634         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5635         unsigned char *b = skb_tail_pointer(skb);
5636         struct nlattr *nest;
5637
5638         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5639         if (nest == NULL)
5640                 goto nla_put_failure;
5641
5642         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5643             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5644                           NFT_DATA_VALUE, set->klen) < 0)
5645                 goto nla_put_failure;
5646
5647         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5648             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5649                           NFT_DATA_VALUE, set->klen) < 0)
5650                 goto nla_put_failure;
5651
5652         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5653             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5654                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5655                           set->dlen) < 0)
5656                 goto nla_put_failure;
5657
5658         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5659             nft_set_elem_expr_dump(skb, set, ext, reset))
5660                 goto nla_put_failure;
5661
5662         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5663             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5664                            (*nft_set_ext_obj(ext))->key.name) < 0)
5665                 goto nla_put_failure;
5666
5667         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5668             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5669                          htonl(*nft_set_ext_flags(ext))))
5670                 goto nla_put_failure;
5671
5672         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
5673             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5674                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
5675                          NFTA_SET_ELEM_PAD))
5676                 goto nla_put_failure;
5677
5678         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5679                 u64 expires, now = get_jiffies_64();
5680
5681                 expires = *nft_set_ext_expiration(ext);
5682                 if (time_before64(now, expires))
5683                         expires -= now;
5684                 else
5685                         expires = 0;
5686
5687                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5688                                  nf_jiffies64_to_msecs(expires),
5689                                  NFTA_SET_ELEM_PAD))
5690                         goto nla_put_failure;
5691         }
5692
5693         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5694                 struct nft_userdata *udata;
5695
5696                 udata = nft_set_ext_userdata(ext);
5697                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5698                             udata->len + 1, udata->data))
5699                         goto nla_put_failure;
5700         }
5701
5702         nla_nest_end(skb, nest);
5703         return 0;
5704
5705 nla_put_failure:
5706         nlmsg_trim(skb, b);
5707         return -EMSGSIZE;
5708 }
5709
5710 struct nft_set_dump_args {
5711         const struct netlink_callback   *cb;
5712         struct nft_set_iter             iter;
5713         struct sk_buff                  *skb;
5714         bool                            reset;
5715 };
5716
5717 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5718                                   struct nft_set *set,
5719                                   const struct nft_set_iter *iter,
5720                                   struct nft_elem_priv *elem_priv)
5721 {
5722         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5723         struct nft_set_dump_args *args;
5724
5725         if (nft_set_elem_expired(ext) || nft_set_elem_is_dead(ext))
5726                 return 0;
5727
5728         args = container_of(iter, struct nft_set_dump_args, iter);
5729         return nf_tables_fill_setelem(args->skb, set, elem_priv, args->reset);
5730 }
5731
5732 static void audit_log_nft_set_reset(const struct nft_table *table,
5733                                     unsigned int base_seq,
5734                                     unsigned int nentries)
5735 {
5736         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
5737
5738         audit_log_nfcfg(buf, table->family, nentries,
5739                         AUDIT_NFT_OP_SETELEM_RESET, GFP_ATOMIC);
5740         kfree(buf);
5741 }
5742
5743 struct nft_set_dump_ctx {
5744         const struct nft_set    *set;
5745         struct nft_ctx          ctx;
5746         bool                    reset;
5747 };
5748
5749 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5750                                  const struct nft_set *set, bool reset,
5751                                  unsigned int base_seq)
5752 {
5753         struct nft_set_elem_catchall *catchall;
5754         u8 genmask = nft_genmask_cur(net);
5755         struct nft_set_ext *ext;
5756         int ret = 0;
5757
5758         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5759                 ext = nft_set_elem_ext(set, catchall->elem);
5760                 if (!nft_set_elem_active(ext, genmask) ||
5761                     nft_set_elem_expired(ext))
5762                         continue;
5763
5764                 ret = nf_tables_fill_setelem(skb, set, catchall->elem, reset);
5765                 if (reset && !ret)
5766                         audit_log_nft_set_reset(set->table, base_seq, 1);
5767                 break;
5768         }
5769
5770         return ret;
5771 }
5772
5773 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5774 {
5775         struct nft_set_dump_ctx *dump_ctx = cb->data;
5776         struct net *net = sock_net(skb->sk);
5777         struct nftables_pernet *nft_net;
5778         struct nft_table *table;
5779         struct nft_set *set;
5780         struct nft_set_dump_args args;
5781         bool set_found = false;
5782         struct nlmsghdr *nlh;
5783         struct nlattr *nest;
5784         u32 portid, seq;
5785         int event;
5786
5787         rcu_read_lock();
5788         nft_net = nft_pernet(net);
5789         cb->seq = READ_ONCE(nft_net->base_seq);
5790
5791         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5792                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5793                     dump_ctx->ctx.family != table->family)
5794                         continue;
5795
5796                 if (table != dump_ctx->ctx.table)
5797                         continue;
5798
5799                 list_for_each_entry_rcu(set, &table->sets, list) {
5800                         if (set == dump_ctx->set) {
5801                                 set_found = true;
5802                                 break;
5803                         }
5804                 }
5805                 break;
5806         }
5807
5808         if (!set_found) {
5809                 rcu_read_unlock();
5810                 return -ENOENT;
5811         }
5812
5813         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5814         portid = NETLINK_CB(cb->skb).portid;
5815         seq    = cb->nlh->nlmsg_seq;
5816
5817         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5818                            table->family, NFNETLINK_V0, nft_base_seq(net));
5819         if (!nlh)
5820                 goto nla_put_failure;
5821
5822         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5823                 goto nla_put_failure;
5824         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5825                 goto nla_put_failure;
5826
5827         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5828         if (nest == NULL)
5829                 goto nla_put_failure;
5830
5831         args.cb                 = cb;
5832         args.skb                = skb;
5833         args.reset              = dump_ctx->reset;
5834         args.iter.genmask       = nft_genmask_cur(net);
5835         args.iter.skip          = cb->args[0];
5836         args.iter.count         = 0;
5837         args.iter.err           = 0;
5838         args.iter.fn            = nf_tables_dump_setelem;
5839         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5840
5841         if (!args.iter.err && args.iter.count == cb->args[0])
5842                 args.iter.err = nft_set_catchall_dump(net, skb, set,
5843                                                       dump_ctx->reset, cb->seq);
5844         nla_nest_end(skb, nest);
5845         nlmsg_end(skb, nlh);
5846
5847         rcu_read_unlock();
5848
5849         if (args.iter.err && args.iter.err != -EMSGSIZE)
5850                 return args.iter.err;
5851         if (args.iter.count == cb->args[0])
5852                 return 0;
5853
5854         cb->args[0] = args.iter.count;
5855         return skb->len;
5856
5857 nla_put_failure:
5858         rcu_read_unlock();
5859         return -ENOSPC;
5860 }
5861
5862 static int nf_tables_dumpreset_set(struct sk_buff *skb,
5863                                    struct netlink_callback *cb)
5864 {
5865         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
5866         struct nft_set_dump_ctx *dump_ctx = cb->data;
5867         int ret, skip = cb->args[0];
5868
5869         mutex_lock(&nft_net->commit_mutex);
5870
5871         ret = nf_tables_dump_set(skb, cb);
5872
5873         if (cb->args[0] > skip)
5874                 audit_log_nft_set_reset(dump_ctx->ctx.table, cb->seq,
5875                                         cb->args[0] - skip);
5876
5877         mutex_unlock(&nft_net->commit_mutex);
5878
5879         return ret;
5880 }
5881
5882 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5883 {
5884         struct nft_set_dump_ctx *dump_ctx = cb->data;
5885
5886         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5887
5888         return cb->data ? 0 : -ENOMEM;
5889 }
5890
5891 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5892 {
5893         kfree(cb->data);
5894         return 0;
5895 }
5896
5897 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5898                                        const struct nft_ctx *ctx, u32 seq,
5899                                        u32 portid, int event, u16 flags,
5900                                        const struct nft_set *set,
5901                                        const struct nft_elem_priv *elem_priv,
5902                                        bool reset)
5903 {
5904         struct nlmsghdr *nlh;
5905         struct nlattr *nest;
5906         int err;
5907
5908         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
5909         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
5910                            NFNETLINK_V0, nft_base_seq(ctx->net));
5911         if (!nlh)
5912                 goto nla_put_failure;
5913
5914         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
5915                 goto nla_put_failure;
5916         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
5917                 goto nla_put_failure;
5918
5919         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5920         if (nest == NULL)
5921                 goto nla_put_failure;
5922
5923         err = nf_tables_fill_setelem(skb, set, elem_priv, reset);
5924         if (err < 0)
5925                 goto nla_put_failure;
5926
5927         nla_nest_end(skb, nest);
5928
5929         nlmsg_end(skb, nlh);
5930         return 0;
5931
5932 nla_put_failure:
5933         nlmsg_trim(skb, nlh);
5934         return -1;
5935 }
5936
5937 static int nft_setelem_parse_flags(const struct nft_set *set,
5938                                    const struct nlattr *attr, u32 *flags)
5939 {
5940         if (attr == NULL)
5941                 return 0;
5942
5943         *flags = ntohl(nla_get_be32(attr));
5944         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5945                 return -EOPNOTSUPP;
5946         if (!(set->flags & NFT_SET_INTERVAL) &&
5947             *flags & NFT_SET_ELEM_INTERVAL_END)
5948                 return -EINVAL;
5949         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
5950             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5951                 return -EINVAL;
5952
5953         return 0;
5954 }
5955
5956 static int nft_setelem_parse_key(struct nft_ctx *ctx, const struct nft_set *set,
5957                                  struct nft_data *key, struct nlattr *attr)
5958 {
5959         struct nft_data_desc desc = {
5960                 .type   = NFT_DATA_VALUE,
5961                 .size   = NFT_DATA_VALUE_MAXLEN,
5962                 .len    = set->klen,
5963         };
5964
5965         return nft_data_init(ctx, key, &desc, attr);
5966 }
5967
5968 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
5969                                   struct nft_data_desc *desc,
5970                                   struct nft_data *data,
5971                                   struct nlattr *attr)
5972 {
5973         u32 dtype;
5974
5975         if (set->dtype == NFT_DATA_VERDICT)
5976                 dtype = NFT_DATA_VERDICT;
5977         else
5978                 dtype = NFT_DATA_VALUE;
5979
5980         desc->type = dtype;
5981         desc->size = NFT_DATA_VALUE_MAXLEN;
5982         desc->len = set->dlen;
5983         desc->flags = NFT_DATA_DESC_SETELEM;
5984
5985         return nft_data_init(ctx, data, desc, attr);
5986 }
5987
5988 static void *nft_setelem_catchall_get(const struct net *net,
5989                                       const struct nft_set *set)
5990 {
5991         struct nft_set_elem_catchall *catchall;
5992         u8 genmask = nft_genmask_cur(net);
5993         struct nft_set_ext *ext;
5994         void *priv = NULL;
5995
5996         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5997                 ext = nft_set_elem_ext(set, catchall->elem);
5998                 if (!nft_set_elem_active(ext, genmask) ||
5999                     nft_set_elem_expired(ext))
6000                         continue;
6001
6002                 priv = catchall->elem;
6003                 break;
6004         }
6005
6006         return priv;
6007 }
6008
6009 static int nft_setelem_get(struct nft_ctx *ctx, const struct nft_set *set,
6010                            struct nft_set_elem *elem, u32 flags)
6011 {
6012         void *priv;
6013
6014         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
6015                 priv = set->ops->get(ctx->net, set, elem, flags);
6016                 if (IS_ERR(priv))
6017                         return PTR_ERR(priv);
6018         } else {
6019                 priv = nft_setelem_catchall_get(ctx->net, set);
6020                 if (!priv)
6021                         return -ENOENT;
6022         }
6023         elem->priv = priv;
6024
6025         return 0;
6026 }
6027
6028 static int nft_get_set_elem(struct nft_ctx *ctx, const struct nft_set *set,
6029                             const struct nlattr *attr, bool reset)
6030 {
6031         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6032         struct nft_set_elem elem;
6033         struct sk_buff *skb;
6034         uint32_t flags = 0;
6035         int err;
6036
6037         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6038                                           nft_set_elem_policy, NULL);
6039         if (err < 0)
6040                 return err;
6041
6042         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6043         if (err < 0)
6044                 return err;
6045
6046         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6047                 return -EINVAL;
6048
6049         if (nla[NFTA_SET_ELEM_KEY]) {
6050                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6051                                             nla[NFTA_SET_ELEM_KEY]);
6052                 if (err < 0)
6053                         return err;
6054         }
6055
6056         if (nla[NFTA_SET_ELEM_KEY_END]) {
6057                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6058                                             nla[NFTA_SET_ELEM_KEY_END]);
6059                 if (err < 0)
6060                         return err;
6061         }
6062
6063         err = nft_setelem_get(ctx, set, &elem, flags);
6064         if (err < 0)
6065                 return err;
6066
6067         err = -ENOMEM;
6068         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
6069         if (skb == NULL)
6070                 return err;
6071
6072         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
6073                                           NFT_MSG_NEWSETELEM, 0, set, elem.priv,
6074                                           reset);
6075         if (err < 0)
6076                 goto err_fill_setelem;
6077
6078         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
6079
6080 err_fill_setelem:
6081         kfree_skb(skb);
6082         return err;
6083 }
6084
6085 static int nft_set_dump_ctx_init(struct nft_set_dump_ctx *dump_ctx,
6086                                  const struct sk_buff *skb,
6087                                  const struct nfnl_info *info,
6088                                  const struct nlattr * const nla[],
6089                                  bool reset)
6090 {
6091         struct netlink_ext_ack *extack = info->extack;
6092         u8 genmask = nft_genmask_cur(info->net);
6093         u8 family = info->nfmsg->nfgen_family;
6094         struct net *net = info->net;
6095         struct nft_table *table;
6096         struct nft_set *set;
6097
6098         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6099                                  genmask, 0);
6100         if (IS_ERR(table)) {
6101                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6102                 return PTR_ERR(table);
6103         }
6104
6105         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6106         if (IS_ERR(set)) {
6107                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
6108                 return PTR_ERR(set);
6109         }
6110
6111         nft_ctx_init(&dump_ctx->ctx, net, skb,
6112                      info->nlh, family, table, NULL, nla);
6113         dump_ctx->set = set;
6114         dump_ctx->reset = reset;
6115         return 0;
6116 }
6117
6118 /* called with rcu_read_lock held */
6119 static int nf_tables_getsetelem(struct sk_buff *skb,
6120                                 const struct nfnl_info *info,
6121                                 const struct nlattr * const nla[])
6122 {
6123         struct netlink_ext_ack *extack = info->extack;
6124         struct nft_set_dump_ctx dump_ctx;
6125         struct nlattr *attr;
6126         int rem, err = 0;
6127
6128         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6129                 struct netlink_dump_control c = {
6130                         .start = nf_tables_dump_set_start,
6131                         .dump = nf_tables_dump_set,
6132                         .done = nf_tables_dump_set_done,
6133                         .module = THIS_MODULE,
6134                 };
6135
6136                 err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false);
6137                 if (err)
6138                         return err;
6139
6140                 c.data = &dump_ctx;
6141                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6142         }
6143
6144         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6145                 return -EINVAL;
6146
6147         err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false);
6148         if (err)
6149                 return err;
6150
6151         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6152                 err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, false);
6153                 if (err < 0) {
6154                         NL_SET_BAD_ATTR(extack, attr);
6155                         break;
6156                 }
6157         }
6158
6159         return err;
6160 }
6161
6162 static int nf_tables_getsetelem_reset(struct sk_buff *skb,
6163                                       const struct nfnl_info *info,
6164                                       const struct nlattr * const nla[])
6165 {
6166         struct nftables_pernet *nft_net = nft_pernet(info->net);
6167         struct netlink_ext_ack *extack = info->extack;
6168         struct nft_set_dump_ctx dump_ctx;
6169         int rem, err = 0, nelems = 0;
6170         struct nlattr *attr;
6171
6172         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6173                 struct netlink_dump_control c = {
6174                         .start = nf_tables_dump_set_start,
6175                         .dump = nf_tables_dumpreset_set,
6176                         .done = nf_tables_dump_set_done,
6177                         .module = THIS_MODULE,
6178                 };
6179
6180                 err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true);
6181                 if (err)
6182                         return err;
6183
6184                 c.data = &dump_ctx;
6185                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6186         }
6187
6188         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6189                 return -EINVAL;
6190
6191         if (!try_module_get(THIS_MODULE))
6192                 return -EINVAL;
6193         rcu_read_unlock();
6194         mutex_lock(&nft_net->commit_mutex);
6195         rcu_read_lock();
6196
6197         err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true);
6198         if (err)
6199                 goto out_unlock;
6200
6201         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6202                 err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, true);
6203                 if (err < 0) {
6204                         NL_SET_BAD_ATTR(extack, attr);
6205                         break;
6206                 }
6207                 nelems++;
6208         }
6209         audit_log_nft_set_reset(dump_ctx.ctx.table, nft_net->base_seq, nelems);
6210
6211 out_unlock:
6212         rcu_read_unlock();
6213         mutex_unlock(&nft_net->commit_mutex);
6214         rcu_read_lock();
6215         module_put(THIS_MODULE);
6216
6217         return err;
6218 }
6219
6220 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
6221                                      const struct nft_set *set,
6222                                      const struct nft_elem_priv *elem_priv,
6223                                      int event)
6224 {
6225         struct nftables_pernet *nft_net;
6226         struct net *net = ctx->net;
6227         u32 portid = ctx->portid;
6228         struct sk_buff *skb;
6229         u16 flags = 0;
6230         int err;
6231
6232         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
6233                 return;
6234
6235         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
6236         if (skb == NULL)
6237                 goto err;
6238
6239         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
6240                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
6241
6242         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
6243                                           set, elem_priv, false);
6244         if (err < 0) {
6245                 kfree_skb(skb);
6246                 goto err;
6247         }
6248
6249         nft_net = nft_pernet(net);
6250         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
6251         return;
6252 err:
6253         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
6254 }
6255
6256 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
6257                                               int msg_type,
6258                                               struct nft_set *set)
6259 {
6260         struct nft_trans *trans;
6261
6262         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
6263         if (trans == NULL)
6264                 return NULL;
6265
6266         nft_trans_elem_set(trans) = set;
6267         return trans;
6268 }
6269
6270 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
6271                                          const struct nft_set *set,
6272                                          const struct nlattr *attr)
6273 {
6274         struct nft_expr *expr;
6275         int err;
6276
6277         expr = nft_expr_init(ctx, attr);
6278         if (IS_ERR(expr))
6279                 return expr;
6280
6281         err = -EOPNOTSUPP;
6282         if (expr->ops->type->flags & NFT_EXPR_GC) {
6283                 if (set->flags & NFT_SET_TIMEOUT)
6284                         goto err_set_elem_expr;
6285                 if (!set->ops->gc_init)
6286                         goto err_set_elem_expr;
6287                 set->ops->gc_init(set);
6288         }
6289
6290         return expr;
6291
6292 err_set_elem_expr:
6293         nft_expr_destroy(ctx, expr);
6294         return ERR_PTR(err);
6295 }
6296
6297 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
6298 {
6299         len += nft_set_ext_types[id].len;
6300         if (len > tmpl->ext_len[id] ||
6301             len > U8_MAX)
6302                 return -1;
6303
6304         return 0;
6305 }
6306
6307 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
6308                               void *to, const void *from, u32 len)
6309 {
6310         if (nft_set_ext_check(tmpl, id, len) < 0)
6311                 return -1;
6312
6313         memcpy(to, from, len);
6314
6315         return 0;
6316 }
6317
6318 struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set,
6319                                         const struct nft_set_ext_tmpl *tmpl,
6320                                         const u32 *key, const u32 *key_end,
6321                                         const u32 *data,
6322                                         u64 timeout, u64 expiration, gfp_t gfp)
6323 {
6324         struct nft_set_ext *ext;
6325         void *elem;
6326
6327         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
6328         if (elem == NULL)
6329                 return ERR_PTR(-ENOMEM);
6330
6331         ext = nft_set_elem_ext(set, elem);
6332         nft_set_ext_init(ext, tmpl);
6333
6334         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
6335             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
6336                                nft_set_ext_key(ext), key, set->klen) < 0)
6337                 goto err_ext_check;
6338
6339         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
6340             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
6341                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
6342                 goto err_ext_check;
6343
6344         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6345             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
6346                                nft_set_ext_data(ext), data, set->dlen) < 0)
6347                 goto err_ext_check;
6348
6349         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
6350                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
6351                 if (expiration == 0)
6352                         *nft_set_ext_expiration(ext) += timeout;
6353         }
6354         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
6355                 *nft_set_ext_timeout(ext) = timeout;
6356
6357         return elem;
6358
6359 err_ext_check:
6360         kfree(elem);
6361
6362         return ERR_PTR(-EINVAL);
6363 }
6364
6365 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6366                                         struct nft_expr *expr)
6367 {
6368         if (expr->ops->destroy_clone) {
6369                 expr->ops->destroy_clone(ctx, expr);
6370                 module_put(expr->ops->type->owner);
6371         } else {
6372                 nf_tables_expr_destroy(ctx, expr);
6373         }
6374 }
6375
6376 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6377                                       struct nft_set_elem_expr *elem_expr)
6378 {
6379         struct nft_expr *expr;
6380         u32 size;
6381
6382         nft_setelem_expr_foreach(expr, elem_expr, size)
6383                 __nft_set_elem_expr_destroy(ctx, expr);
6384 }
6385
6386 /* Drop references and destroy. Called from gc, dynset and abort path. */
6387 void nft_set_elem_destroy(const struct nft_set *set,
6388                           const struct nft_elem_priv *elem_priv,
6389                           bool destroy_expr)
6390 {
6391         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6392         struct nft_ctx ctx = {
6393                 .net    = read_pnet(&set->net),
6394                 .family = set->table->family,
6395         };
6396
6397         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
6398         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6399                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6400         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6401                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
6402         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6403                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
6404
6405         kfree(elem_priv);
6406 }
6407 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
6408
6409 /* Destroy element. References have been already dropped in the preparation
6410  * path via nft_setelem_data_deactivate().
6411  */
6412 void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
6413                                 const struct nft_set *set,
6414                                 const struct nft_elem_priv *elem_priv)
6415 {
6416         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6417
6418         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6419                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6420
6421         kfree(elem_priv);
6422 }
6423
6424 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
6425                             struct nft_expr *expr_array[])
6426 {
6427         struct nft_expr *expr;
6428         int err, i, k;
6429
6430         for (i = 0; i < set->num_exprs; i++) {
6431                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
6432                 if (!expr)
6433                         goto err_expr;
6434
6435                 err = nft_expr_clone(expr, set->exprs[i]);
6436                 if (err < 0) {
6437                         kfree(expr);
6438                         goto err_expr;
6439                 }
6440                 expr_array[i] = expr;
6441         }
6442
6443         return 0;
6444
6445 err_expr:
6446         for (k = i - 1; k >= 0; k--)
6447                 nft_expr_destroy(ctx, expr_array[k]);
6448
6449         return -ENOMEM;
6450 }
6451
6452 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
6453                                    const struct nft_set_ext_tmpl *tmpl,
6454                                    const struct nft_set_ext *ext,
6455                                    struct nft_expr *expr_array[],
6456                                    u32 num_exprs)
6457 {
6458         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
6459         u32 len = sizeof(struct nft_set_elem_expr);
6460         struct nft_expr *expr;
6461         int i, err;
6462
6463         if (num_exprs == 0)
6464                 return 0;
6465
6466         for (i = 0; i < num_exprs; i++)
6467                 len += expr_array[i]->ops->size;
6468
6469         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
6470                 return -EINVAL;
6471
6472         for (i = 0; i < num_exprs; i++) {
6473                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
6474                 err = nft_expr_clone(expr, expr_array[i]);
6475                 if (err < 0)
6476                         goto err_elem_expr_setup;
6477
6478                 elem_expr->size += expr_array[i]->ops->size;
6479                 nft_expr_destroy(ctx, expr_array[i]);
6480                 expr_array[i] = NULL;
6481         }
6482
6483         return 0;
6484
6485 err_elem_expr_setup:
6486         for (; i < num_exprs; i++) {
6487                 nft_expr_destroy(ctx, expr_array[i]);
6488                 expr_array[i] = NULL;
6489         }
6490
6491         return -ENOMEM;
6492 }
6493
6494 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6495                                             const struct nft_set *set)
6496 {
6497         struct nft_set_elem_catchall *catchall;
6498         u8 genmask = nft_genmask_cur(net);
6499         struct nft_set_ext *ext;
6500
6501         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6502                 ext = nft_set_elem_ext(set, catchall->elem);
6503                 if (nft_set_elem_active(ext, genmask) &&
6504                     !nft_set_elem_expired(ext) &&
6505                     !nft_set_elem_is_dead(ext))
6506                         return ext;
6507         }
6508
6509         return NULL;
6510 }
6511 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6512
6513 static int nft_setelem_catchall_insert(const struct net *net,
6514                                        struct nft_set *set,
6515                                        const struct nft_set_elem *elem,
6516                                        struct nft_elem_priv **priv)
6517 {
6518         struct nft_set_elem_catchall *catchall;
6519         u8 genmask = nft_genmask_next(net);
6520         struct nft_set_ext *ext;
6521
6522         list_for_each_entry(catchall, &set->catchall_list, list) {
6523                 ext = nft_set_elem_ext(set, catchall->elem);
6524                 if (nft_set_elem_active(ext, genmask)) {
6525                         *priv = catchall->elem;
6526                         return -EEXIST;
6527                 }
6528         }
6529
6530         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
6531         if (!catchall)
6532                 return -ENOMEM;
6533
6534         catchall->elem = elem->priv;
6535         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6536
6537         return 0;
6538 }
6539
6540 static int nft_setelem_insert(const struct net *net,
6541                               struct nft_set *set,
6542                               const struct nft_set_elem *elem,
6543                               struct nft_elem_priv **elem_priv,
6544                               unsigned int flags)
6545 {
6546         int ret;
6547
6548         if (flags & NFT_SET_ELEM_CATCHALL)
6549                 ret = nft_setelem_catchall_insert(net, set, elem, elem_priv);
6550         else
6551                 ret = set->ops->insert(net, set, elem, elem_priv);
6552
6553         return ret;
6554 }
6555
6556 static bool nft_setelem_is_catchall(const struct nft_set *set,
6557                                     const struct nft_elem_priv *elem_priv)
6558 {
6559         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6560
6561         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6562             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6563                 return true;
6564
6565         return false;
6566 }
6567
6568 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6569                                  struct nft_elem_priv *elem_priv)
6570 {
6571         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6572
6573         if (nft_setelem_is_catchall(set, elem_priv)) {
6574                 nft_set_elem_change_active(net, set, ext);
6575         } else {
6576                 set->ops->activate(net, set, elem_priv);
6577         }
6578 }
6579
6580 static int nft_setelem_catchall_deactivate(const struct net *net,
6581                                            struct nft_set *set,
6582                                            struct nft_set_elem *elem)
6583 {
6584         struct nft_set_elem_catchall *catchall;
6585         struct nft_set_ext *ext;
6586
6587         list_for_each_entry(catchall, &set->catchall_list, list) {
6588                 ext = nft_set_elem_ext(set, catchall->elem);
6589                 if (!nft_is_active_next(net, ext))
6590                         continue;
6591
6592                 kfree(elem->priv);
6593                 elem->priv = catchall->elem;
6594                 nft_set_elem_change_active(net, set, ext);
6595                 return 0;
6596         }
6597
6598         return -ENOENT;
6599 }
6600
6601 static int __nft_setelem_deactivate(const struct net *net,
6602                                     struct nft_set *set,
6603                                     struct nft_set_elem *elem)
6604 {
6605         void *priv;
6606
6607         priv = set->ops->deactivate(net, set, elem);
6608         if (!priv)
6609                 return -ENOENT;
6610
6611         kfree(elem->priv);
6612         elem->priv = priv;
6613         set->ndeact++;
6614
6615         return 0;
6616 }
6617
6618 static int nft_setelem_deactivate(const struct net *net,
6619                                   struct nft_set *set,
6620                                   struct nft_set_elem *elem, u32 flags)
6621 {
6622         int ret;
6623
6624         if (flags & NFT_SET_ELEM_CATCHALL)
6625                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6626         else
6627                 ret = __nft_setelem_deactivate(net, set, elem);
6628
6629         return ret;
6630 }
6631
6632 static void nft_setelem_catchall_destroy(struct nft_set_elem_catchall *catchall)
6633 {
6634         list_del_rcu(&catchall->list);
6635         kfree_rcu(catchall, rcu);
6636 }
6637
6638 static void nft_setelem_catchall_remove(const struct net *net,
6639                                         const struct nft_set *set,
6640                                         struct nft_elem_priv *elem_priv)
6641 {
6642         struct nft_set_elem_catchall *catchall, *next;
6643
6644         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6645                 if (catchall->elem == elem_priv) {
6646                         nft_setelem_catchall_destroy(catchall);
6647                         break;
6648                 }
6649         }
6650 }
6651
6652 static void nft_setelem_remove(const struct net *net,
6653                                const struct nft_set *set,
6654                                struct nft_elem_priv *elem_priv)
6655 {
6656         if (nft_setelem_is_catchall(set, elem_priv))
6657                 nft_setelem_catchall_remove(net, set, elem_priv);
6658         else
6659                 set->ops->remove(net, set, elem_priv);
6660 }
6661
6662 static bool nft_setelem_valid_key_end(const struct nft_set *set,
6663                                       struct nlattr **nla, u32 flags)
6664 {
6665         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
6666                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
6667                 if (flags & NFT_SET_ELEM_INTERVAL_END)
6668                         return false;
6669
6670                 if (nla[NFTA_SET_ELEM_KEY_END] &&
6671                     flags & NFT_SET_ELEM_CATCHALL)
6672                         return false;
6673         } else {
6674                 if (nla[NFTA_SET_ELEM_KEY_END])
6675                         return false;
6676         }
6677
6678         return true;
6679 }
6680
6681 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
6682                             const struct nlattr *attr, u32 nlmsg_flags)
6683 {
6684         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
6685         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6686         u8 genmask = nft_genmask_next(ctx->net);
6687         u32 flags = 0, size = 0, num_exprs = 0;
6688         struct nft_set_ext_tmpl tmpl;
6689         struct nft_set_ext *ext, *ext2;
6690         struct nft_set_elem elem;
6691         struct nft_set_binding *binding;
6692         struct nft_elem_priv *elem_priv;
6693         struct nft_object *obj = NULL;
6694         struct nft_userdata *udata;
6695         struct nft_data_desc desc;
6696         enum nft_registers dreg;
6697         struct nft_trans *trans;
6698         u64 expiration;
6699         u64 timeout;
6700         int err, i;
6701         u8 ulen;
6702
6703         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6704                                           nft_set_elem_policy, NULL);
6705         if (err < 0)
6706                 return err;
6707
6708         nft_set_ext_prepare(&tmpl);
6709
6710         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6711         if (err < 0)
6712                 return err;
6713
6714         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
6715             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
6716                 return -EINVAL;
6717
6718         if (flags != 0) {
6719                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6720                 if (err < 0)
6721                         return err;
6722         }
6723
6724         if (set->flags & NFT_SET_MAP) {
6725                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6726                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6727                         return -EINVAL;
6728         } else {
6729                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6730                         return -EINVAL;
6731         }
6732
6733         if (set->flags & NFT_SET_OBJECT) {
6734                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6735                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6736                         return -EINVAL;
6737         } else {
6738                 if (nla[NFTA_SET_ELEM_OBJREF])
6739                         return -EINVAL;
6740         }
6741
6742         if (!nft_setelem_valid_key_end(set, nla, flags))
6743                 return -EINVAL;
6744
6745         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6746              (nla[NFTA_SET_ELEM_DATA] ||
6747               nla[NFTA_SET_ELEM_OBJREF] ||
6748               nla[NFTA_SET_ELEM_TIMEOUT] ||
6749               nla[NFTA_SET_ELEM_EXPIRATION] ||
6750               nla[NFTA_SET_ELEM_USERDATA] ||
6751               nla[NFTA_SET_ELEM_EXPR] ||
6752               nla[NFTA_SET_ELEM_KEY_END] ||
6753               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6754                 return -EINVAL;
6755
6756         timeout = 0;
6757         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6758                 if (!(set->flags & NFT_SET_TIMEOUT))
6759                         return -EINVAL;
6760                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6761                                             &timeout);
6762                 if (err)
6763                         return err;
6764         } else if (set->flags & NFT_SET_TIMEOUT &&
6765                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6766                 timeout = READ_ONCE(set->timeout);
6767         }
6768
6769         expiration = 0;
6770         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6771                 if (!(set->flags & NFT_SET_TIMEOUT))
6772                         return -EINVAL;
6773                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6774                                             &expiration);
6775                 if (err)
6776                         return err;
6777         }
6778
6779         if (nla[NFTA_SET_ELEM_EXPR]) {
6780                 struct nft_expr *expr;
6781
6782                 if (set->num_exprs && set->num_exprs != 1)
6783                         return -EOPNOTSUPP;
6784
6785                 expr = nft_set_elem_expr_alloc(ctx, set,
6786                                                nla[NFTA_SET_ELEM_EXPR]);
6787                 if (IS_ERR(expr))
6788                         return PTR_ERR(expr);
6789
6790                 expr_array[0] = expr;
6791                 num_exprs = 1;
6792
6793                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6794                         err = -EOPNOTSUPP;
6795                         goto err_set_elem_expr;
6796                 }
6797         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6798                 struct nft_expr *expr;
6799                 struct nlattr *tmp;
6800                 int left;
6801
6802                 i = 0;
6803                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6804                         if (i == NFT_SET_EXPR_MAX ||
6805                             (set->num_exprs && set->num_exprs == i)) {
6806                                 err = -E2BIG;
6807                                 goto err_set_elem_expr;
6808                         }
6809                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6810                                 err = -EINVAL;
6811                                 goto err_set_elem_expr;
6812                         }
6813                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6814                         if (IS_ERR(expr)) {
6815                                 err = PTR_ERR(expr);
6816                                 goto err_set_elem_expr;
6817                         }
6818                         expr_array[i] = expr;
6819                         num_exprs++;
6820
6821                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6822                                 err = -EOPNOTSUPP;
6823                                 goto err_set_elem_expr;
6824                         }
6825                         i++;
6826                 }
6827                 if (set->num_exprs && set->num_exprs != i) {
6828                         err = -EOPNOTSUPP;
6829                         goto err_set_elem_expr;
6830                 }
6831         } else if (set->num_exprs > 0 &&
6832                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6833                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6834                 if (err < 0)
6835                         goto err_set_elem_expr_clone;
6836
6837                 num_exprs = set->num_exprs;
6838         }
6839
6840         if (nla[NFTA_SET_ELEM_KEY]) {
6841                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6842                                             nla[NFTA_SET_ELEM_KEY]);
6843                 if (err < 0)
6844                         goto err_set_elem_expr;
6845
6846                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6847                 if (err < 0)
6848                         goto err_parse_key;
6849         }
6850
6851         if (nla[NFTA_SET_ELEM_KEY_END]) {
6852                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6853                                             nla[NFTA_SET_ELEM_KEY_END]);
6854                 if (err < 0)
6855                         goto err_parse_key;
6856
6857                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6858                 if (err < 0)
6859                         goto err_parse_key_end;
6860         }
6861
6862         if (timeout > 0) {
6863                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6864                 if (err < 0)
6865                         goto err_parse_key_end;
6866
6867                 if (timeout != READ_ONCE(set->timeout)) {
6868                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6869                         if (err < 0)
6870                                 goto err_parse_key_end;
6871                 }
6872         }
6873
6874         if (num_exprs) {
6875                 for (i = 0; i < num_exprs; i++)
6876                         size += expr_array[i]->ops->size;
6877
6878                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6879                                              sizeof(struct nft_set_elem_expr) + size);
6880                 if (err < 0)
6881                         goto err_parse_key_end;
6882         }
6883
6884         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6885                 obj = nft_obj_lookup(ctx->net, ctx->table,
6886                                      nla[NFTA_SET_ELEM_OBJREF],
6887                                      set->objtype, genmask);
6888                 if (IS_ERR(obj)) {
6889                         err = PTR_ERR(obj);
6890                         obj = NULL;
6891                         goto err_parse_key_end;
6892                 }
6893
6894                 if (!nft_use_inc(&obj->use)) {
6895                         err = -EMFILE;
6896                         obj = NULL;
6897                         goto err_parse_key_end;
6898                 }
6899
6900                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6901                 if (err < 0)
6902                         goto err_parse_key_end;
6903         }
6904
6905         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6906                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
6907                                              nla[NFTA_SET_ELEM_DATA]);
6908                 if (err < 0)
6909                         goto err_parse_key_end;
6910
6911                 dreg = nft_type_to_reg(set->dtype);
6912                 list_for_each_entry(binding, &set->bindings, list) {
6913                         struct nft_ctx bind_ctx = {
6914                                 .net    = ctx->net,
6915                                 .family = ctx->family,
6916                                 .table  = ctx->table,
6917                                 .chain  = (struct nft_chain *)binding->chain,
6918                         };
6919
6920                         if (!(binding->flags & NFT_SET_MAP))
6921                                 continue;
6922
6923                         err = nft_validate_register_store(&bind_ctx, dreg,
6924                                                           &elem.data.val,
6925                                                           desc.type, desc.len);
6926                         if (err < 0)
6927                                 goto err_parse_data;
6928
6929                         if (desc.type == NFT_DATA_VERDICT &&
6930                             (elem.data.val.verdict.code == NFT_GOTO ||
6931                              elem.data.val.verdict.code == NFT_JUMP))
6932                                 nft_validate_state_update(ctx->table,
6933                                                           NFT_VALIDATE_NEED);
6934                 }
6935
6936                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
6937                 if (err < 0)
6938                         goto err_parse_data;
6939         }
6940
6941         /* The full maximum length of userdata can exceed the maximum
6942          * offset value (U8_MAX) for following extensions, therefor it
6943          * must be the last extension added.
6944          */
6945         ulen = 0;
6946         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
6947                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
6948                 if (ulen > 0) {
6949                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
6950                                                      ulen);
6951                         if (err < 0)
6952                                 goto err_parse_data;
6953                 }
6954         }
6955
6956         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6957                                       elem.key_end.val.data, elem.data.val.data,
6958                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
6959         if (IS_ERR(elem.priv)) {
6960                 err = PTR_ERR(elem.priv);
6961                 goto err_parse_data;
6962         }
6963
6964         ext = nft_set_elem_ext(set, elem.priv);
6965         if (flags)
6966                 *nft_set_ext_flags(ext) = flags;
6967
6968         if (obj)
6969                 *nft_set_ext_obj(ext) = obj;
6970
6971         if (ulen > 0) {
6972                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
6973                         err = -EINVAL;
6974                         goto err_elem_free;
6975                 }
6976                 udata = nft_set_ext_userdata(ext);
6977                 udata->len = ulen - 1;
6978                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
6979         }
6980         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
6981         if (err < 0)
6982                 goto err_elem_free;
6983
6984         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
6985         if (trans == NULL) {
6986                 err = -ENOMEM;
6987                 goto err_elem_free;
6988         }
6989
6990         ext->genmask = nft_genmask_cur(ctx->net);
6991
6992         err = nft_setelem_insert(ctx->net, set, &elem, &elem_priv, flags);
6993         if (err) {
6994                 if (err == -EEXIST) {
6995                         ext2 = nft_set_elem_ext(set, elem_priv);
6996                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
6997                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
6998                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
6999                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
7000                                 goto err_element_clash;
7001                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
7002                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
7003                              memcmp(nft_set_ext_data(ext),
7004                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
7005                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
7006                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
7007                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
7008                                 goto err_element_clash;
7009                         else if (!(nlmsg_flags & NLM_F_EXCL))
7010                                 err = 0;
7011                 } else if (err == -ENOTEMPTY) {
7012                         /* ENOTEMPTY reports overlapping between this element
7013                          * and an existing one.
7014                          */
7015                         err = -EEXIST;
7016                 }
7017                 goto err_element_clash;
7018         }
7019
7020         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
7021                 unsigned int max = set->size ? set->size + set->ndeact : UINT_MAX;
7022
7023                 if (!atomic_add_unless(&set->nelems, 1, max)) {
7024                         err = -ENFILE;
7025                         goto err_set_full;
7026                 }
7027         }
7028
7029         nft_trans_elem_priv(trans) = elem.priv;
7030         nft_trans_commit_list_add_tail(ctx->net, trans);
7031         return 0;
7032
7033 err_set_full:
7034         nft_setelem_remove(ctx->net, set, elem.priv);
7035 err_element_clash:
7036         kfree(trans);
7037 err_elem_free:
7038         nf_tables_set_elem_destroy(ctx, set, elem.priv);
7039 err_parse_data:
7040         if (nla[NFTA_SET_ELEM_DATA] != NULL)
7041                 nft_data_release(&elem.data.val, desc.type);
7042 err_parse_key_end:
7043         if (obj)
7044                 nft_use_dec_restore(&obj->use);
7045
7046         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7047 err_parse_key:
7048         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7049 err_set_elem_expr:
7050         for (i = 0; i < num_exprs && expr_array[i]; i++)
7051                 nft_expr_destroy(ctx, expr_array[i]);
7052 err_set_elem_expr_clone:
7053         return err;
7054 }
7055
7056 static int nf_tables_newsetelem(struct sk_buff *skb,
7057                                 const struct nfnl_info *info,
7058                                 const struct nlattr * const nla[])
7059 {
7060         struct netlink_ext_ack *extack = info->extack;
7061         u8 genmask = nft_genmask_next(info->net);
7062         u8 family = info->nfmsg->nfgen_family;
7063         struct net *net = info->net;
7064         const struct nlattr *attr;
7065         struct nft_table *table;
7066         struct nft_set *set;
7067         struct nft_ctx ctx;
7068         int rem, err;
7069
7070         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
7071                 return -EINVAL;
7072
7073         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7074                                  genmask, NETLINK_CB(skb).portid);
7075         if (IS_ERR(table)) {
7076                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7077                 return PTR_ERR(table);
7078         }
7079
7080         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
7081                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
7082         if (IS_ERR(set)) {
7083                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
7084                 return PTR_ERR(set);
7085         }
7086
7087         if (!list_empty(&set->bindings) &&
7088             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
7089                 return -EBUSY;
7090
7091         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7092
7093         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7094                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
7095                 if (err < 0) {
7096                         NL_SET_BAD_ATTR(extack, attr);
7097                         return err;
7098                 }
7099         }
7100
7101         if (table->validate_state == NFT_VALIDATE_DO)
7102                 return nft_table_validate(net, table);
7103
7104         return 0;
7105 }
7106
7107 /**
7108  *      nft_data_hold - hold a nft_data item
7109  *
7110  *      @data: struct nft_data to release
7111  *      @type: type of data
7112  *
7113  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
7114  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
7115  *      NFT_GOTO verdicts. This function must be called on active data objects
7116  *      from the second phase of the commit protocol.
7117  */
7118 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
7119 {
7120         struct nft_chain *chain;
7121
7122         if (type == NFT_DATA_VERDICT) {
7123                 switch (data->verdict.code) {
7124                 case NFT_JUMP:
7125                 case NFT_GOTO:
7126                         chain = data->verdict.chain;
7127                         nft_use_inc_restore(&chain->use);
7128                         break;
7129                 }
7130         }
7131 }
7132
7133 static void nft_setelem_data_activate(const struct net *net,
7134                                       const struct nft_set *set,
7135                                       struct nft_elem_priv *elem_priv)
7136 {
7137         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7138
7139         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7140                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
7141         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7142                 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
7143 }
7144
7145 void nft_setelem_data_deactivate(const struct net *net,
7146                                  const struct nft_set *set,
7147                                  struct nft_elem_priv *elem_priv)
7148 {
7149         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7150
7151         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7152                 nft_data_release(nft_set_ext_data(ext), set->dtype);
7153         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7154                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
7155 }
7156
7157 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
7158                            const struct nlattr *attr)
7159 {
7160         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
7161         struct nft_set_ext_tmpl tmpl;
7162         struct nft_set_elem elem;
7163         struct nft_set_ext *ext;
7164         struct nft_trans *trans;
7165         u32 flags = 0;
7166         int err;
7167
7168         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
7169                                           nft_set_elem_policy, NULL);
7170         if (err < 0)
7171                 return err;
7172
7173         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
7174         if (err < 0)
7175                 return err;
7176
7177         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
7178                 return -EINVAL;
7179
7180         if (!nft_setelem_valid_key_end(set, nla, flags))
7181                 return -EINVAL;
7182
7183         nft_set_ext_prepare(&tmpl);
7184
7185         if (flags != 0) {
7186                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
7187                 if (err < 0)
7188                         return err;
7189         }
7190
7191         if (nla[NFTA_SET_ELEM_KEY]) {
7192                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
7193                                             nla[NFTA_SET_ELEM_KEY]);
7194                 if (err < 0)
7195                         return err;
7196
7197                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
7198                 if (err < 0)
7199                         goto fail_elem;
7200         }
7201
7202         if (nla[NFTA_SET_ELEM_KEY_END]) {
7203                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
7204                                             nla[NFTA_SET_ELEM_KEY_END]);
7205                 if (err < 0)
7206                         goto fail_elem;
7207
7208                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
7209                 if (err < 0)
7210                         goto fail_elem_key_end;
7211         }
7212
7213         err = -ENOMEM;
7214         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7215                                       elem.key_end.val.data, NULL, 0, 0,
7216                                       GFP_KERNEL_ACCOUNT);
7217         if (IS_ERR(elem.priv)) {
7218                 err = PTR_ERR(elem.priv);
7219                 goto fail_elem_key_end;
7220         }
7221
7222         ext = nft_set_elem_ext(set, elem.priv);
7223         if (flags)
7224                 *nft_set_ext_flags(ext) = flags;
7225
7226         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
7227         if (trans == NULL)
7228                 goto fail_trans;
7229
7230         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
7231         if (err < 0)
7232                 goto fail_ops;
7233
7234         nft_setelem_data_deactivate(ctx->net, set, elem.priv);
7235
7236         nft_trans_elem_priv(trans) = elem.priv;
7237         nft_trans_commit_list_add_tail(ctx->net, trans);
7238         return 0;
7239
7240 fail_ops:
7241         kfree(trans);
7242 fail_trans:
7243         kfree(elem.priv);
7244 fail_elem_key_end:
7245         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7246 fail_elem:
7247         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7248         return err;
7249 }
7250
7251 static int nft_setelem_flush(const struct nft_ctx *ctx,
7252                              struct nft_set *set,
7253                              const struct nft_set_iter *iter,
7254                              struct nft_elem_priv *elem_priv)
7255 {
7256         struct nft_trans *trans;
7257
7258         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7259                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
7260         if (!trans)
7261                 return -ENOMEM;
7262
7263         set->ops->flush(ctx->net, set, elem_priv);
7264         set->ndeact++;
7265
7266         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
7267         nft_trans_elem_set(trans) = set;
7268         nft_trans_elem_priv(trans) = elem_priv;
7269         nft_trans_commit_list_add_tail(ctx->net, trans);
7270
7271         return 0;
7272 }
7273
7274 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
7275                                     struct nft_set *set,
7276                                     struct nft_elem_priv *elem_priv)
7277 {
7278         struct nft_trans *trans;
7279
7280         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7281                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
7282         if (!trans)
7283                 return -ENOMEM;
7284
7285         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
7286         nft_trans_elem_set(trans) = set;
7287         nft_trans_elem_priv(trans) = elem_priv;
7288         nft_trans_commit_list_add_tail(ctx->net, trans);
7289
7290         return 0;
7291 }
7292
7293 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
7294                                   struct nft_set *set)
7295 {
7296         u8 genmask = nft_genmask_next(ctx->net);
7297         struct nft_set_elem_catchall *catchall;
7298         struct nft_set_ext *ext;
7299         int ret = 0;
7300
7301         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
7302                 ext = nft_set_elem_ext(set, catchall->elem);
7303                 if (!nft_set_elem_active(ext, genmask))
7304                         continue;
7305
7306                 ret = __nft_set_catchall_flush(ctx, set, catchall->elem);
7307                 if (ret < 0)
7308                         break;
7309                 nft_set_elem_change_active(ctx->net, set, ext);
7310         }
7311
7312         return ret;
7313 }
7314
7315 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
7316 {
7317         struct nft_set_iter iter = {
7318                 .genmask        = genmask,
7319                 .fn             = nft_setelem_flush,
7320         };
7321
7322         set->ops->walk(ctx, set, &iter);
7323         if (!iter.err)
7324                 iter.err = nft_set_catchall_flush(ctx, set);
7325
7326         return iter.err;
7327 }
7328
7329 static int nf_tables_delsetelem(struct sk_buff *skb,
7330                                 const struct nfnl_info *info,
7331                                 const struct nlattr * const nla[])
7332 {
7333         struct netlink_ext_ack *extack = info->extack;
7334         u8 genmask = nft_genmask_next(info->net);
7335         u8 family = info->nfmsg->nfgen_family;
7336         struct net *net = info->net;
7337         const struct nlattr *attr;
7338         struct nft_table *table;
7339         struct nft_set *set;
7340         struct nft_ctx ctx;
7341         int rem, err = 0;
7342
7343         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7344                                  genmask, NETLINK_CB(skb).portid);
7345         if (IS_ERR(table)) {
7346                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7347                 return PTR_ERR(table);
7348         }
7349
7350         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
7351         if (IS_ERR(set)) {
7352                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
7353                 return PTR_ERR(set);
7354         }
7355
7356         if (nft_set_is_anonymous(set))
7357                 return -EOPNOTSUPP;
7358
7359         if (!list_empty(&set->bindings) && (set->flags & NFT_SET_CONSTANT))
7360                 return -EBUSY;
7361
7362         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7363
7364         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
7365                 return nft_set_flush(&ctx, set, genmask);
7366
7367         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7368                 err = nft_del_setelem(&ctx, set, attr);
7369                 if (err == -ENOENT &&
7370                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
7371                         continue;
7372
7373                 if (err < 0) {
7374                         NL_SET_BAD_ATTR(extack, attr);
7375                         return err;
7376                 }
7377         }
7378
7379         return 0;
7380 }
7381
7382 /*
7383  * Stateful objects
7384  */
7385
7386 /**
7387  *      nft_register_obj- register nf_tables stateful object type
7388  *      @obj_type: object type
7389  *
7390  *      Registers the object type for use with nf_tables. Returns zero on
7391  *      success or a negative errno code otherwise.
7392  */
7393 int nft_register_obj(struct nft_object_type *obj_type)
7394 {
7395         if (obj_type->type == NFT_OBJECT_UNSPEC)
7396                 return -EINVAL;
7397
7398         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7399         list_add_rcu(&obj_type->list, &nf_tables_objects);
7400         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7401         return 0;
7402 }
7403 EXPORT_SYMBOL_GPL(nft_register_obj);
7404
7405 /**
7406  *      nft_unregister_obj - unregister nf_tables object type
7407  *      @obj_type: object type
7408  *
7409  *      Unregisters the object type for use with nf_tables.
7410  */
7411 void nft_unregister_obj(struct nft_object_type *obj_type)
7412 {
7413         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7414         list_del_rcu(&obj_type->list);
7415         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7416 }
7417 EXPORT_SYMBOL_GPL(nft_unregister_obj);
7418
7419 struct nft_object *nft_obj_lookup(const struct net *net,
7420                                   const struct nft_table *table,
7421                                   const struct nlattr *nla, u32 objtype,
7422                                   u8 genmask)
7423 {
7424         struct nft_object_hash_key k = { .table = table };
7425         char search[NFT_OBJ_MAXNAMELEN];
7426         struct rhlist_head *tmp, *list;
7427         struct nft_object *obj;
7428
7429         nla_strscpy(search, nla, sizeof(search));
7430         k.name = search;
7431
7432         WARN_ON_ONCE(!rcu_read_lock_held() &&
7433                      !lockdep_commit_lock_is_held(net));
7434
7435         rcu_read_lock();
7436         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
7437         if (!list)
7438                 goto out;
7439
7440         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
7441                 if (objtype == obj->ops->type->type &&
7442                     nft_active_genmask(obj, genmask)) {
7443                         rcu_read_unlock();
7444                         return obj;
7445                 }
7446         }
7447 out:
7448         rcu_read_unlock();
7449         return ERR_PTR(-ENOENT);
7450 }
7451 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7452
7453 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7454                                                   const struct nlattr *nla,
7455                                                   u32 objtype, u8 genmask)
7456 {
7457         struct nft_object *obj;
7458
7459         list_for_each_entry(obj, &table->objects, list) {
7460                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7461                     objtype == obj->ops->type->type &&
7462                     nft_active_genmask(obj, genmask))
7463                         return obj;
7464         }
7465         return ERR_PTR(-ENOENT);
7466 }
7467
7468 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7469         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7470                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7471         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7472                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7473         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7474         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7475         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7476         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7477                                     .len = NFT_USERDATA_MAXLEN },
7478 };
7479
7480 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7481                                        const struct nft_object_type *type,
7482                                        const struct nlattr *attr)
7483 {
7484         struct nlattr **tb;
7485         const struct nft_object_ops *ops;
7486         struct nft_object *obj;
7487         int err = -ENOMEM;
7488
7489         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7490         if (!tb)
7491                 goto err1;
7492
7493         if (attr) {
7494                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7495                                                   type->policy, NULL);
7496                 if (err < 0)
7497                         goto err2;
7498         } else {
7499                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7500         }
7501
7502         if (type->select_ops) {
7503                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7504                 if (IS_ERR(ops)) {
7505                         err = PTR_ERR(ops);
7506                         goto err2;
7507                 }
7508         } else {
7509                 ops = type->ops;
7510         }
7511
7512         err = -ENOMEM;
7513         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7514         if (!obj)
7515                 goto err2;
7516
7517         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7518         if (err < 0)
7519                 goto err3;
7520
7521         obj->ops = ops;
7522
7523         kfree(tb);
7524         return obj;
7525 err3:
7526         kfree(obj);
7527 err2:
7528         kfree(tb);
7529 err1:
7530         return ERR_PTR(err);
7531 }
7532
7533 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
7534                            struct nft_object *obj, bool reset)
7535 {
7536         struct nlattr *nest;
7537
7538         nest = nla_nest_start_noflag(skb, attr);
7539         if (!nest)
7540                 goto nla_put_failure;
7541         if (obj->ops->dump(skb, obj, reset) < 0)
7542                 goto nla_put_failure;
7543         nla_nest_end(skb, nest);
7544         return 0;
7545
7546 nla_put_failure:
7547         return -1;
7548 }
7549
7550 static const struct nft_object_type *__nft_obj_type_get(u32 objtype)
7551 {
7552         const struct nft_object_type *type;
7553
7554         list_for_each_entry(type, &nf_tables_objects, list) {
7555                 if (objtype == type->type)
7556                         return type;
7557         }
7558         return NULL;
7559 }
7560
7561 static const struct nft_object_type *
7562 nft_obj_type_get(struct net *net, u32 objtype)
7563 {
7564         const struct nft_object_type *type;
7565
7566         type = __nft_obj_type_get(objtype);
7567         if (type != NULL && try_module_get(type->owner))
7568                 return type;
7569
7570         lockdep_nfnl_nft_mutex_not_held();
7571 #ifdef CONFIG_MODULES
7572         if (type == NULL) {
7573                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
7574                         return ERR_PTR(-EAGAIN);
7575         }
7576 #endif
7577         return ERR_PTR(-ENOENT);
7578 }
7579
7580 static int nf_tables_updobj(const struct nft_ctx *ctx,
7581                             const struct nft_object_type *type,
7582                             const struct nlattr *attr,
7583                             struct nft_object *obj)
7584 {
7585         struct nft_object *newobj;
7586         struct nft_trans *trans;
7587         int err = -ENOMEM;
7588
7589         if (!try_module_get(type->owner))
7590                 return -ENOENT;
7591
7592         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
7593                                 sizeof(struct nft_trans_obj));
7594         if (!trans)
7595                 goto err_trans;
7596
7597         newobj = nft_obj_init(ctx, type, attr);
7598         if (IS_ERR(newobj)) {
7599                 err = PTR_ERR(newobj);
7600                 goto err_free_trans;
7601         }
7602
7603         nft_trans_obj(trans) = obj;
7604         nft_trans_obj_update(trans) = true;
7605         nft_trans_obj_newobj(trans) = newobj;
7606         nft_trans_commit_list_add_tail(ctx->net, trans);
7607
7608         return 0;
7609
7610 err_free_trans:
7611         kfree(trans);
7612 err_trans:
7613         module_put(type->owner);
7614         return err;
7615 }
7616
7617 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
7618                             const struct nlattr * const nla[])
7619 {
7620         struct netlink_ext_ack *extack = info->extack;
7621         u8 genmask = nft_genmask_next(info->net);
7622         u8 family = info->nfmsg->nfgen_family;
7623         const struct nft_object_type *type;
7624         struct net *net = info->net;
7625         struct nft_table *table;
7626         struct nft_object *obj;
7627         struct nft_ctx ctx;
7628         u32 objtype;
7629         int err;
7630
7631         if (!nla[NFTA_OBJ_TYPE] ||
7632             !nla[NFTA_OBJ_NAME] ||
7633             !nla[NFTA_OBJ_DATA])
7634                 return -EINVAL;
7635
7636         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7637                                  NETLINK_CB(skb).portid);
7638         if (IS_ERR(table)) {
7639                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7640                 return PTR_ERR(table);
7641         }
7642
7643         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7644         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7645         if (IS_ERR(obj)) {
7646                 err = PTR_ERR(obj);
7647                 if (err != -ENOENT) {
7648                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7649                         return err;
7650                 }
7651         } else {
7652                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7653                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7654                         return -EEXIST;
7655                 }
7656                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
7657                         return -EOPNOTSUPP;
7658
7659                 type = __nft_obj_type_get(objtype);
7660                 if (WARN_ON_ONCE(!type))
7661                         return -ENOENT;
7662
7663                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7664
7665                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
7666         }
7667
7668         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7669
7670         if (!nft_use_inc(&table->use))
7671                 return -EMFILE;
7672
7673         type = nft_obj_type_get(net, objtype);
7674         if (IS_ERR(type)) {
7675                 err = PTR_ERR(type);
7676                 goto err_type;
7677         }
7678
7679         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
7680         if (IS_ERR(obj)) {
7681                 err = PTR_ERR(obj);
7682                 goto err_init;
7683         }
7684         obj->key.table = table;
7685         obj->handle = nf_tables_alloc_handle(table);
7686
7687         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
7688         if (!obj->key.name) {
7689                 err = -ENOMEM;
7690                 goto err_strdup;
7691         }
7692
7693         if (nla[NFTA_OBJ_USERDATA]) {
7694                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
7695                 if (obj->udata == NULL)
7696                         goto err_userdata;
7697
7698                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
7699         }
7700
7701         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
7702         if (err < 0)
7703                 goto err_trans;
7704
7705         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
7706                               nft_objname_ht_params);
7707         if (err < 0)
7708                 goto err_obj_ht;
7709
7710         list_add_tail_rcu(&obj->list, &table->objects);
7711
7712         return 0;
7713 err_obj_ht:
7714         /* queued in transaction log */
7715         INIT_LIST_HEAD(&obj->list);
7716         return err;
7717 err_trans:
7718         kfree(obj->udata);
7719 err_userdata:
7720         kfree(obj->key.name);
7721 err_strdup:
7722         if (obj->ops->destroy)
7723                 obj->ops->destroy(&ctx, obj);
7724         kfree(obj);
7725 err_init:
7726         module_put(type->owner);
7727 err_type:
7728         nft_use_dec_restore(&table->use);
7729
7730         return err;
7731 }
7732
7733 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7734                                    u32 portid, u32 seq, int event, u32 flags,
7735                                    int family, const struct nft_table *table,
7736                                    struct nft_object *obj, bool reset)
7737 {
7738         struct nlmsghdr *nlh;
7739
7740         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7741         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7742                            NFNETLINK_V0, nft_base_seq(net));
7743         if (!nlh)
7744                 goto nla_put_failure;
7745
7746         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7747             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7748             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7749                          NFTA_OBJ_PAD))
7750                 goto nla_put_failure;
7751
7752         if (event == NFT_MSG_DELOBJ) {
7753                 nlmsg_end(skb, nlh);
7754                 return 0;
7755         }
7756
7757         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7758             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7759             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
7760                 goto nla_put_failure;
7761
7762         if (obj->udata &&
7763             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7764                 goto nla_put_failure;
7765
7766         nlmsg_end(skb, nlh);
7767         return 0;
7768
7769 nla_put_failure:
7770         nlmsg_trim(skb, nlh);
7771         return -1;
7772 }
7773
7774 static void audit_log_obj_reset(const struct nft_table *table,
7775                                 unsigned int base_seq, unsigned int nentries)
7776 {
7777         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
7778
7779         audit_log_nfcfg(buf, table->family, nentries,
7780                         AUDIT_NFT_OP_OBJ_RESET, GFP_ATOMIC);
7781         kfree(buf);
7782 }
7783
7784 struct nft_obj_dump_ctx {
7785         unsigned int    s_idx;
7786         char            *table;
7787         u32             type;
7788         bool            reset;
7789 };
7790
7791 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7792 {
7793         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7794         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7795         struct net *net = sock_net(skb->sk);
7796         int family = nfmsg->nfgen_family;
7797         struct nftables_pernet *nft_net;
7798         const struct nft_table *table;
7799         unsigned int entries = 0;
7800         struct nft_object *obj;
7801         unsigned int idx = 0;
7802         int rc = 0;
7803
7804         rcu_read_lock();
7805         nft_net = nft_pernet(net);
7806         cb->seq = READ_ONCE(nft_net->base_seq);
7807
7808         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7809                 if (family != NFPROTO_UNSPEC && family != table->family)
7810                         continue;
7811
7812                 entries = 0;
7813                 list_for_each_entry_rcu(obj, &table->objects, list) {
7814                         if (!nft_is_active(net, obj))
7815                                 goto cont;
7816                         if (idx < ctx->s_idx)
7817                                 goto cont;
7818                         if (ctx->table && strcmp(ctx->table, table->name))
7819                                 goto cont;
7820                         if (ctx->type != NFT_OBJECT_UNSPEC &&
7821                             obj->ops->type->type != ctx->type)
7822                                 goto cont;
7823
7824                         rc = nf_tables_fill_obj_info(skb, net,
7825                                                      NETLINK_CB(cb->skb).portid,
7826                                                      cb->nlh->nlmsg_seq,
7827                                                      NFT_MSG_NEWOBJ,
7828                                                      NLM_F_MULTI | NLM_F_APPEND,
7829                                                      table->family, table,
7830                                                      obj, ctx->reset);
7831                         if (rc < 0)
7832                                 break;
7833
7834                         entries++;
7835                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7836 cont:
7837                         idx++;
7838                 }
7839                 if (ctx->reset && entries)
7840                         audit_log_obj_reset(table, nft_net->base_seq, entries);
7841                 if (rc < 0)
7842                         break;
7843         }
7844         rcu_read_unlock();
7845
7846         ctx->s_idx = idx;
7847         return skb->len;
7848 }
7849
7850 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7851 {
7852         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7853         const struct nlattr * const *nla = cb->data;
7854
7855         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
7856
7857         if (nla[NFTA_OBJ_TABLE]) {
7858                 ctx->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7859                 if (!ctx->table)
7860                         return -ENOMEM;
7861         }
7862
7863         if (nla[NFTA_OBJ_TYPE])
7864                 ctx->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7865
7866         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7867                 ctx->reset = true;
7868
7869         return 0;
7870 }
7871
7872 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7873 {
7874         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7875
7876         kfree(ctx->table);
7877
7878         return 0;
7879 }
7880
7881 /* called with rcu_read_lock held */
7882 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7883                             const struct nlattr * const nla[])
7884 {
7885         struct netlink_ext_ack *extack = info->extack;
7886         u8 genmask = nft_genmask_cur(info->net);
7887         u8 family = info->nfmsg->nfgen_family;
7888         const struct nft_table *table;
7889         struct net *net = info->net;
7890         struct nft_object *obj;
7891         struct sk_buff *skb2;
7892         bool reset = false;
7893         u32 objtype;
7894         int err;
7895
7896         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
7897                 struct netlink_dump_control c = {
7898                         .start = nf_tables_dump_obj_start,
7899                         .dump = nf_tables_dump_obj,
7900                         .done = nf_tables_dump_obj_done,
7901                         .module = THIS_MODULE,
7902                         .data = (void *)nla,
7903                 };
7904
7905                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
7906         }
7907
7908         if (!nla[NFTA_OBJ_NAME] ||
7909             !nla[NFTA_OBJ_TYPE])
7910                 return -EINVAL;
7911
7912         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
7913         if (IS_ERR(table)) {
7914                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7915                 return PTR_ERR(table);
7916         }
7917
7918         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7919         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7920         if (IS_ERR(obj)) {
7921                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7922                 return PTR_ERR(obj);
7923         }
7924
7925         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7926         if (!skb2)
7927                 return -ENOMEM;
7928
7929         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7930                 reset = true;
7931
7932         if (reset) {
7933                 const struct nftables_pernet *nft_net;
7934                 char *buf;
7935
7936                 nft_net = nft_pernet(net);
7937                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
7938
7939                 audit_log_nfcfg(buf,
7940                                 family,
7941                                 1,
7942                                 AUDIT_NFT_OP_OBJ_RESET,
7943                                 GFP_ATOMIC);
7944                 kfree(buf);
7945         }
7946
7947         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
7948                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
7949                                       family, table, obj, reset);
7950         if (err < 0)
7951                 goto err_fill_obj_info;
7952
7953         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
7954
7955 err_fill_obj_info:
7956         kfree_skb(skb2);
7957         return err;
7958 }
7959
7960 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
7961 {
7962         if (obj->ops->destroy)
7963                 obj->ops->destroy(ctx, obj);
7964
7965         module_put(obj->ops->type->owner);
7966         kfree(obj->key.name);
7967         kfree(obj->udata);
7968         kfree(obj);
7969 }
7970
7971 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
7972                             const struct nlattr * const nla[])
7973 {
7974         struct netlink_ext_ack *extack = info->extack;
7975         u8 genmask = nft_genmask_next(info->net);
7976         u8 family = info->nfmsg->nfgen_family;
7977         struct net *net = info->net;
7978         const struct nlattr *attr;
7979         struct nft_table *table;
7980         struct nft_object *obj;
7981         struct nft_ctx ctx;
7982         u32 objtype;
7983
7984         if (!nla[NFTA_OBJ_TYPE] ||
7985             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
7986                 return -EINVAL;
7987
7988         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7989                                  NETLINK_CB(skb).portid);
7990         if (IS_ERR(table)) {
7991                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7992                 return PTR_ERR(table);
7993         }
7994
7995         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7996         if (nla[NFTA_OBJ_HANDLE]) {
7997                 attr = nla[NFTA_OBJ_HANDLE];
7998                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
7999         } else {
8000                 attr = nla[NFTA_OBJ_NAME];
8001                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
8002         }
8003
8004         if (IS_ERR(obj)) {
8005                 if (PTR_ERR(obj) == -ENOENT &&
8006                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
8007                         return 0;
8008
8009                 NL_SET_BAD_ATTR(extack, attr);
8010                 return PTR_ERR(obj);
8011         }
8012         if (obj->use > 0) {
8013                 NL_SET_BAD_ATTR(extack, attr);
8014                 return -EBUSY;
8015         }
8016
8017         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8018
8019         return nft_delobj(&ctx, obj);
8020 }
8021
8022 static void
8023 __nft_obj_notify(struct net *net, const struct nft_table *table,
8024                  struct nft_object *obj, u32 portid, u32 seq, int event,
8025                  u16 flags, int family, int report, gfp_t gfp)
8026 {
8027         struct nftables_pernet *nft_net = nft_pernet(net);
8028         struct sk_buff *skb;
8029         int err;
8030
8031         if (!report &&
8032             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8033                 return;
8034
8035         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
8036         if (skb == NULL)
8037                 goto err;
8038
8039         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
8040                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
8041                                       family, table, obj, false);
8042         if (err < 0) {
8043                 kfree_skb(skb);
8044                 goto err;
8045         }
8046
8047         nft_notify_enqueue(skb, report, &nft_net->notify_list);
8048         return;
8049 err:
8050         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
8051 }
8052
8053 void nft_obj_notify(struct net *net, const struct nft_table *table,
8054                     struct nft_object *obj, u32 portid, u32 seq, int event,
8055                     u16 flags, int family, int report, gfp_t gfp)
8056 {
8057         struct nftables_pernet *nft_net = nft_pernet(net);
8058         char *buf = kasprintf(gfp, "%s:%u",
8059                               table->name, nft_net->base_seq);
8060
8061         audit_log_nfcfg(buf,
8062                         family,
8063                         obj->handle,
8064                         event == NFT_MSG_NEWOBJ ?
8065                                  AUDIT_NFT_OP_OBJ_REGISTER :
8066                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
8067                         gfp);
8068         kfree(buf);
8069
8070         __nft_obj_notify(net, table, obj, portid, seq, event,
8071                          flags, family, report, gfp);
8072 }
8073 EXPORT_SYMBOL_GPL(nft_obj_notify);
8074
8075 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
8076                                  struct nft_object *obj, int event)
8077 {
8078         __nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid,
8079                          ctx->seq, event, ctx->flags, ctx->family,
8080                          ctx->report, GFP_KERNEL);
8081 }
8082
8083 /*
8084  * Flow tables
8085  */
8086 void nft_register_flowtable_type(struct nf_flowtable_type *type)
8087 {
8088         nfnl_lock(NFNL_SUBSYS_NFTABLES);
8089         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
8090         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
8091 }
8092 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
8093
8094 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
8095 {
8096         nfnl_lock(NFNL_SUBSYS_NFTABLES);
8097         list_del_rcu(&type->list);
8098         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
8099 }
8100 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
8101
8102 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
8103         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
8104                                             .len = NFT_NAME_MAXLEN - 1 },
8105         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
8106                                             .len = NFT_NAME_MAXLEN - 1 },
8107         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
8108         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
8109         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
8110 };
8111
8112 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
8113                                            const struct nlattr *nla, u8 genmask)
8114 {
8115         struct nft_flowtable *flowtable;
8116
8117         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8118                 if (!nla_strcmp(nla, flowtable->name) &&
8119                     nft_active_genmask(flowtable, genmask))
8120                         return flowtable;
8121         }
8122         return ERR_PTR(-ENOENT);
8123 }
8124 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
8125
8126 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
8127                                     struct nft_flowtable *flowtable,
8128                                     enum nft_trans_phase phase)
8129 {
8130         switch (phase) {
8131         case NFT_TRANS_PREPARE_ERROR:
8132         case NFT_TRANS_PREPARE:
8133         case NFT_TRANS_ABORT:
8134         case NFT_TRANS_RELEASE:
8135                 nft_use_dec(&flowtable->use);
8136                 fallthrough;
8137         default:
8138                 return;
8139         }
8140 }
8141 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
8142
8143 static struct nft_flowtable *
8144 nft_flowtable_lookup_byhandle(const struct nft_table *table,
8145                               const struct nlattr *nla, u8 genmask)
8146 {
8147        struct nft_flowtable *flowtable;
8148
8149        list_for_each_entry(flowtable, &table->flowtables, list) {
8150                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
8151                    nft_active_genmask(flowtable, genmask))
8152                        return flowtable;
8153        }
8154        return ERR_PTR(-ENOENT);
8155 }
8156
8157 struct nft_flowtable_hook {
8158         u32                     num;
8159         int                     priority;
8160         struct list_head        list;
8161 };
8162
8163 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
8164         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
8165         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
8166         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
8167 };
8168
8169 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
8170                                     const struct nlattr * const nla[],
8171                                     struct nft_flowtable_hook *flowtable_hook,
8172                                     struct nft_flowtable *flowtable,
8173                                     struct netlink_ext_ack *extack, bool add)
8174 {
8175         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
8176         struct nft_hook *hook;
8177         int hooknum, priority;
8178         int err;
8179
8180         INIT_LIST_HEAD(&flowtable_hook->list);
8181
8182         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
8183                                           nla[NFTA_FLOWTABLE_HOOK],
8184                                           nft_flowtable_hook_policy, NULL);
8185         if (err < 0)
8186                 return err;
8187
8188         if (add) {
8189                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
8190                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8191                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8192                         return -ENOENT;
8193                 }
8194
8195                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8196                 if (hooknum != NF_NETDEV_INGRESS)
8197                         return -EOPNOTSUPP;
8198
8199                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8200
8201                 flowtable_hook->priority        = priority;
8202                 flowtable_hook->num             = hooknum;
8203         } else {
8204                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
8205                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8206                         if (hooknum != flowtable->hooknum)
8207                                 return -EOPNOTSUPP;
8208                 }
8209
8210                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8211                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8212                         if (priority != flowtable->data.priority)
8213                                 return -EOPNOTSUPP;
8214                 }
8215
8216                 flowtable_hook->priority        = flowtable->data.priority;
8217                 flowtable_hook->num             = flowtable->hooknum;
8218         }
8219
8220         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
8221                 err = nf_tables_parse_netdev_hooks(ctx->net,
8222                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
8223                                                    &flowtable_hook->list,
8224                                                    extack);
8225                 if (err < 0)
8226                         return err;
8227         }
8228
8229         list_for_each_entry(hook, &flowtable_hook->list, list) {
8230                 hook->ops.pf            = NFPROTO_NETDEV;
8231                 hook->ops.hooknum       = flowtable_hook->num;
8232                 hook->ops.priority      = flowtable_hook->priority;
8233                 hook->ops.priv          = &flowtable->data;
8234                 hook->ops.hook          = flowtable->data.type->hook;
8235         }
8236
8237         return err;
8238 }
8239
8240 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
8241 {
8242         const struct nf_flowtable_type *type;
8243
8244         list_for_each_entry(type, &nf_tables_flowtables, list) {
8245                 if (family == type->family)
8246                         return type;
8247         }
8248         return NULL;
8249 }
8250
8251 static const struct nf_flowtable_type *
8252 nft_flowtable_type_get(struct net *net, u8 family)
8253 {
8254         const struct nf_flowtable_type *type;
8255
8256         type = __nft_flowtable_type_get(family);
8257         if (type != NULL && try_module_get(type->owner))
8258                 return type;
8259
8260         lockdep_nfnl_nft_mutex_not_held();
8261 #ifdef CONFIG_MODULES
8262         if (type == NULL) {
8263                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
8264                         return ERR_PTR(-EAGAIN);
8265         }
8266 #endif
8267         return ERR_PTR(-ENOENT);
8268 }
8269
8270 /* Only called from error and netdev event paths. */
8271 static void nft_unregister_flowtable_hook(struct net *net,
8272                                           struct nft_flowtable *flowtable,
8273                                           struct nft_hook *hook)
8274 {
8275         nf_unregister_net_hook(net, &hook->ops);
8276         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8277                                     FLOW_BLOCK_UNBIND);
8278 }
8279
8280 static void __nft_unregister_flowtable_net_hooks(struct net *net,
8281                                                  struct list_head *hook_list,
8282                                                  bool release_netdev)
8283 {
8284         struct nft_hook *hook, *next;
8285
8286         list_for_each_entry_safe(hook, next, hook_list, list) {
8287                 nf_unregister_net_hook(net, &hook->ops);
8288                 if (release_netdev) {
8289                         list_del(&hook->list);
8290                         kfree_rcu(hook, rcu);
8291                 }
8292         }
8293 }
8294
8295 static void nft_unregister_flowtable_net_hooks(struct net *net,
8296                                                struct list_head *hook_list)
8297 {
8298         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
8299 }
8300
8301 static int nft_register_flowtable_net_hooks(struct net *net,
8302                                             struct nft_table *table,
8303                                             struct list_head *hook_list,
8304                                             struct nft_flowtable *flowtable)
8305 {
8306         struct nft_hook *hook, *hook2, *next;
8307         struct nft_flowtable *ft;
8308         int err, i = 0;
8309
8310         list_for_each_entry(hook, hook_list, list) {
8311                 list_for_each_entry(ft, &table->flowtables, list) {
8312                         if (!nft_is_active_next(net, ft))
8313                                 continue;
8314
8315                         list_for_each_entry(hook2, &ft->hook_list, list) {
8316                                 if (hook->ops.dev == hook2->ops.dev &&
8317                                     hook->ops.pf == hook2->ops.pf) {
8318                                         err = -EEXIST;
8319                                         goto err_unregister_net_hooks;
8320                                 }
8321                         }
8322                 }
8323
8324                 err = flowtable->data.type->setup(&flowtable->data,
8325                                                   hook->ops.dev,
8326                                                   FLOW_BLOCK_BIND);
8327                 if (err < 0)
8328                         goto err_unregister_net_hooks;
8329
8330                 err = nf_register_net_hook(net, &hook->ops);
8331                 if (err < 0) {
8332                         flowtable->data.type->setup(&flowtable->data,
8333                                                     hook->ops.dev,
8334                                                     FLOW_BLOCK_UNBIND);
8335                         goto err_unregister_net_hooks;
8336                 }
8337
8338                 i++;
8339         }
8340
8341         return 0;
8342
8343 err_unregister_net_hooks:
8344         list_for_each_entry_safe(hook, next, hook_list, list) {
8345                 if (i-- <= 0)
8346                         break;
8347
8348                 nft_unregister_flowtable_hook(net, flowtable, hook);
8349                 list_del_rcu(&hook->list);
8350                 kfree_rcu(hook, rcu);
8351         }
8352
8353         return err;
8354 }
8355
8356 static void nft_hooks_destroy(struct list_head *hook_list)
8357 {
8358         struct nft_hook *hook, *next;
8359
8360         list_for_each_entry_safe(hook, next, hook_list, list) {
8361                 list_del_rcu(&hook->list);
8362                 kfree_rcu(hook, rcu);
8363         }
8364 }
8365
8366 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
8367                                 struct nft_flowtable *flowtable,
8368                                 struct netlink_ext_ack *extack)
8369 {
8370         const struct nlattr * const *nla = ctx->nla;
8371         struct nft_flowtable_hook flowtable_hook;
8372         struct nft_hook *hook, *next;
8373         struct nft_trans *trans;
8374         bool unregister = false;
8375         u32 flags;
8376         int err;
8377
8378         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8379                                        extack, false);
8380         if (err < 0)
8381                 return err;
8382
8383         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8384                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
8385                         list_del(&hook->list);
8386                         kfree(hook);
8387                 }
8388         }
8389
8390         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8391                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8392                 if (flags & ~NFT_FLOWTABLE_MASK) {
8393                         err = -EOPNOTSUPP;
8394                         goto err_flowtable_update_hook;
8395                 }
8396                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
8397                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
8398                         err = -EOPNOTSUPP;
8399                         goto err_flowtable_update_hook;
8400                 }
8401         } else {
8402                 flags = flowtable->data.flags;
8403         }
8404
8405         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
8406                                                &flowtable_hook.list, flowtable);
8407         if (err < 0)
8408                 goto err_flowtable_update_hook;
8409
8410         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
8411                                 sizeof(struct nft_trans_flowtable));
8412         if (!trans) {
8413                 unregister = true;
8414                 err = -ENOMEM;
8415                 goto err_flowtable_update_hook;
8416         }
8417
8418         nft_trans_flowtable_flags(trans) = flags;
8419         nft_trans_flowtable(trans) = flowtable;
8420         nft_trans_flowtable_update(trans) = true;
8421         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8422         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
8423
8424         nft_trans_commit_list_add_tail(ctx->net, trans);
8425
8426         return 0;
8427
8428 err_flowtable_update_hook:
8429         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8430                 if (unregister)
8431                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
8432                 list_del_rcu(&hook->list);
8433                 kfree_rcu(hook, rcu);
8434         }
8435
8436         return err;
8437
8438 }
8439
8440 static int nf_tables_newflowtable(struct sk_buff *skb,
8441                                   const struct nfnl_info *info,
8442                                   const struct nlattr * const nla[])
8443 {
8444         struct netlink_ext_ack *extack = info->extack;
8445         struct nft_flowtable_hook flowtable_hook;
8446         u8 genmask = nft_genmask_next(info->net);
8447         u8 family = info->nfmsg->nfgen_family;
8448         const struct nf_flowtable_type *type;
8449         struct nft_flowtable *flowtable;
8450         struct nft_hook *hook, *next;
8451         struct net *net = info->net;
8452         struct nft_table *table;
8453         struct nft_ctx ctx;
8454         int err;
8455
8456         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8457             !nla[NFTA_FLOWTABLE_NAME] ||
8458             !nla[NFTA_FLOWTABLE_HOOK])
8459                 return -EINVAL;
8460
8461         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8462                                  genmask, NETLINK_CB(skb).portid);
8463         if (IS_ERR(table)) {
8464                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8465                 return PTR_ERR(table);
8466         }
8467
8468         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8469                                          genmask);
8470         if (IS_ERR(flowtable)) {
8471                 err = PTR_ERR(flowtable);
8472                 if (err != -ENOENT) {
8473                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8474                         return err;
8475                 }
8476         } else {
8477                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8478                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8479                         return -EEXIST;
8480                 }
8481
8482                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8483
8484                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
8485         }
8486
8487         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8488
8489         if (!nft_use_inc(&table->use))
8490                 return -EMFILE;
8491
8492         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
8493         if (!flowtable) {
8494                 err = -ENOMEM;
8495                 goto flowtable_alloc;
8496         }
8497
8498         flowtable->table = table;
8499         flowtable->handle = nf_tables_alloc_handle(table);
8500         INIT_LIST_HEAD(&flowtable->hook_list);
8501
8502         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
8503         if (!flowtable->name) {
8504                 err = -ENOMEM;
8505                 goto err1;
8506         }
8507
8508         type = nft_flowtable_type_get(net, family);
8509         if (IS_ERR(type)) {
8510                 err = PTR_ERR(type);
8511                 goto err2;
8512         }
8513
8514         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8515                 flowtable->data.flags =
8516                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8517                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
8518                         err = -EOPNOTSUPP;
8519                         goto err3;
8520                 }
8521         }
8522
8523         write_pnet(&flowtable->data.net, net);
8524         flowtable->data.type = type;
8525         err = type->init(&flowtable->data);
8526         if (err < 0)
8527                 goto err3;
8528
8529         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
8530                                        extack, true);
8531         if (err < 0)
8532                 goto err4;
8533
8534         list_splice(&flowtable_hook.list, &flowtable->hook_list);
8535         flowtable->data.priority = flowtable_hook.priority;
8536         flowtable->hooknum = flowtable_hook.num;
8537
8538         err = nft_register_flowtable_net_hooks(ctx.net, table,
8539                                                &flowtable->hook_list,
8540                                                flowtable);
8541         if (err < 0) {
8542                 nft_hooks_destroy(&flowtable->hook_list);
8543                 goto err4;
8544         }
8545
8546         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
8547         if (err < 0)
8548                 goto err5;
8549
8550         list_add_tail_rcu(&flowtable->list, &table->flowtables);
8551
8552         return 0;
8553 err5:
8554         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8555                 nft_unregister_flowtable_hook(net, flowtable, hook);
8556                 list_del_rcu(&hook->list);
8557                 kfree_rcu(hook, rcu);
8558         }
8559 err4:
8560         flowtable->data.type->free(&flowtable->data);
8561 err3:
8562         module_put(type->owner);
8563 err2:
8564         kfree(flowtable->name);
8565 err1:
8566         kfree(flowtable);
8567 flowtable_alloc:
8568         nft_use_dec_restore(&table->use);
8569
8570         return err;
8571 }
8572
8573 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
8574 {
8575         struct nft_hook *this, *next;
8576
8577         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
8578                 list_del(&this->list);
8579                 kfree(this);
8580         }
8581 }
8582
8583 static int nft_delflowtable_hook(struct nft_ctx *ctx,
8584                                  struct nft_flowtable *flowtable,
8585                                  struct netlink_ext_ack *extack)
8586 {
8587         const struct nlattr * const *nla = ctx->nla;
8588         struct nft_flowtable_hook flowtable_hook;
8589         LIST_HEAD(flowtable_del_list);
8590         struct nft_hook *this, *hook;
8591         struct nft_trans *trans;
8592         int err;
8593
8594         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8595                                        extack, false);
8596         if (err < 0)
8597                 return err;
8598
8599         list_for_each_entry(this, &flowtable_hook.list, list) {
8600                 hook = nft_hook_list_find(&flowtable->hook_list, this);
8601                 if (!hook) {
8602                         err = -ENOENT;
8603                         goto err_flowtable_del_hook;
8604                 }
8605                 list_move(&hook->list, &flowtable_del_list);
8606         }
8607
8608         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
8609                                 sizeof(struct nft_trans_flowtable));
8610         if (!trans) {
8611                 err = -ENOMEM;
8612                 goto err_flowtable_del_hook;
8613         }
8614
8615         nft_trans_flowtable(trans) = flowtable;
8616         nft_trans_flowtable_update(trans) = true;
8617         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8618         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
8619         nft_flowtable_hook_release(&flowtable_hook);
8620
8621         nft_trans_commit_list_add_tail(ctx->net, trans);
8622
8623         return 0;
8624
8625 err_flowtable_del_hook:
8626         list_splice(&flowtable_del_list, &flowtable->hook_list);
8627         nft_flowtable_hook_release(&flowtable_hook);
8628
8629         return err;
8630 }
8631
8632 static int nf_tables_delflowtable(struct sk_buff *skb,
8633                                   const struct nfnl_info *info,
8634                                   const struct nlattr * const nla[])
8635 {
8636         struct netlink_ext_ack *extack = info->extack;
8637         u8 genmask = nft_genmask_next(info->net);
8638         u8 family = info->nfmsg->nfgen_family;
8639         struct nft_flowtable *flowtable;
8640         struct net *net = info->net;
8641         const struct nlattr *attr;
8642         struct nft_table *table;
8643         struct nft_ctx ctx;
8644
8645         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8646             (!nla[NFTA_FLOWTABLE_NAME] &&
8647              !nla[NFTA_FLOWTABLE_HANDLE]))
8648                 return -EINVAL;
8649
8650         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8651                                  genmask, NETLINK_CB(skb).portid);
8652         if (IS_ERR(table)) {
8653                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8654                 return PTR_ERR(table);
8655         }
8656
8657         if (nla[NFTA_FLOWTABLE_HANDLE]) {
8658                 attr = nla[NFTA_FLOWTABLE_HANDLE];
8659                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
8660         } else {
8661                 attr = nla[NFTA_FLOWTABLE_NAME];
8662                 flowtable = nft_flowtable_lookup(table, attr, genmask);
8663         }
8664
8665         if (IS_ERR(flowtable)) {
8666                 if (PTR_ERR(flowtable) == -ENOENT &&
8667                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
8668                         return 0;
8669
8670                 NL_SET_BAD_ATTR(extack, attr);
8671                 return PTR_ERR(flowtable);
8672         }
8673
8674         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8675
8676         if (nla[NFTA_FLOWTABLE_HOOK])
8677                 return nft_delflowtable_hook(&ctx, flowtable, extack);
8678
8679         if (flowtable->use > 0) {
8680                 NL_SET_BAD_ATTR(extack, attr);
8681                 return -EBUSY;
8682         }
8683
8684         return nft_delflowtable(&ctx, flowtable);
8685 }
8686
8687 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
8688                                          u32 portid, u32 seq, int event,
8689                                          u32 flags, int family,
8690                                          struct nft_flowtable *flowtable,
8691                                          struct list_head *hook_list)
8692 {
8693         struct nlattr *nest, *nest_devs;
8694         struct nft_hook *hook;
8695         struct nlmsghdr *nlh;
8696
8697         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8698         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8699                            NFNETLINK_V0, nft_base_seq(net));
8700         if (!nlh)
8701                 goto nla_put_failure;
8702
8703         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
8704             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
8705             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
8706                          NFTA_FLOWTABLE_PAD))
8707                 goto nla_put_failure;
8708
8709         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
8710                 nlmsg_end(skb, nlh);
8711                 return 0;
8712         }
8713
8714         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
8715             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
8716                 goto nla_put_failure;
8717
8718         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
8719         if (!nest)
8720                 goto nla_put_failure;
8721         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
8722             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
8723                 goto nla_put_failure;
8724
8725         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
8726         if (!nest_devs)
8727                 goto nla_put_failure;
8728
8729         if (!hook_list)
8730                 hook_list = &flowtable->hook_list;
8731
8732         list_for_each_entry_rcu(hook, hook_list, list) {
8733                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
8734                         goto nla_put_failure;
8735         }
8736         nla_nest_end(skb, nest_devs);
8737         nla_nest_end(skb, nest);
8738
8739         nlmsg_end(skb, nlh);
8740         return 0;
8741
8742 nla_put_failure:
8743         nlmsg_trim(skb, nlh);
8744         return -1;
8745 }
8746
8747 struct nft_flowtable_filter {
8748         char            *table;
8749 };
8750
8751 static int nf_tables_dump_flowtable(struct sk_buff *skb,
8752                                     struct netlink_callback *cb)
8753 {
8754         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8755         struct nft_flowtable_filter *filter = cb->data;
8756         unsigned int idx = 0, s_idx = cb->args[0];
8757         struct net *net = sock_net(skb->sk);
8758         int family = nfmsg->nfgen_family;
8759         struct nft_flowtable *flowtable;
8760         struct nftables_pernet *nft_net;
8761         const struct nft_table *table;
8762
8763         rcu_read_lock();
8764         nft_net = nft_pernet(net);
8765         cb->seq = READ_ONCE(nft_net->base_seq);
8766
8767         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8768                 if (family != NFPROTO_UNSPEC && family != table->family)
8769                         continue;
8770
8771                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8772                         if (!nft_is_active(net, flowtable))
8773                                 goto cont;
8774                         if (idx < s_idx)
8775                                 goto cont;
8776                         if (idx > s_idx)
8777                                 memset(&cb->args[1], 0,
8778                                        sizeof(cb->args) - sizeof(cb->args[0]));
8779                         if (filter && filter->table &&
8780                             strcmp(filter->table, table->name))
8781                                 goto cont;
8782
8783                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8784                                                           cb->nlh->nlmsg_seq,
8785                                                           NFT_MSG_NEWFLOWTABLE,
8786                                                           NLM_F_MULTI | NLM_F_APPEND,
8787                                                           table->family,
8788                                                           flowtable, NULL) < 0)
8789                                 goto done;
8790
8791                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8792 cont:
8793                         idx++;
8794                 }
8795         }
8796 done:
8797         rcu_read_unlock();
8798
8799         cb->args[0] = idx;
8800         return skb->len;
8801 }
8802
8803 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8804 {
8805         const struct nlattr * const *nla = cb->data;
8806         struct nft_flowtable_filter *filter = NULL;
8807
8808         if (nla[NFTA_FLOWTABLE_TABLE]) {
8809                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8810                 if (!filter)
8811                         return -ENOMEM;
8812
8813                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8814                                            GFP_ATOMIC);
8815                 if (!filter->table) {
8816                         kfree(filter);
8817                         return -ENOMEM;
8818                 }
8819         }
8820
8821         cb->data = filter;
8822         return 0;
8823 }
8824
8825 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8826 {
8827         struct nft_flowtable_filter *filter = cb->data;
8828
8829         if (!filter)
8830                 return 0;
8831
8832         kfree(filter->table);
8833         kfree(filter);
8834
8835         return 0;
8836 }
8837
8838 /* called with rcu_read_lock held */
8839 static int nf_tables_getflowtable(struct sk_buff *skb,
8840                                   const struct nfnl_info *info,
8841                                   const struct nlattr * const nla[])
8842 {
8843         struct netlink_ext_ack *extack = info->extack;
8844         u8 genmask = nft_genmask_cur(info->net);
8845         u8 family = info->nfmsg->nfgen_family;
8846         struct nft_flowtable *flowtable;
8847         const struct nft_table *table;
8848         struct net *net = info->net;
8849         struct sk_buff *skb2;
8850         int err;
8851
8852         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8853                 struct netlink_dump_control c = {
8854                         .start = nf_tables_dump_flowtable_start,
8855                         .dump = nf_tables_dump_flowtable,
8856                         .done = nf_tables_dump_flowtable_done,
8857                         .module = THIS_MODULE,
8858                         .data = (void *)nla,
8859                 };
8860
8861                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8862         }
8863
8864         if (!nla[NFTA_FLOWTABLE_NAME])
8865                 return -EINVAL;
8866
8867         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8868                                  genmask, 0);
8869         if (IS_ERR(table)) {
8870                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8871                 return PTR_ERR(table);
8872         }
8873
8874         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8875                                          genmask);
8876         if (IS_ERR(flowtable)) {
8877                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8878                 return PTR_ERR(flowtable);
8879         }
8880
8881         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8882         if (!skb2)
8883                 return -ENOMEM;
8884
8885         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
8886                                             info->nlh->nlmsg_seq,
8887                                             NFT_MSG_NEWFLOWTABLE, 0, family,
8888                                             flowtable, NULL);
8889         if (err < 0)
8890                 goto err_fill_flowtable_info;
8891
8892         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8893
8894 err_fill_flowtable_info:
8895         kfree_skb(skb2);
8896         return err;
8897 }
8898
8899 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
8900                                        struct nft_flowtable *flowtable,
8901                                        struct list_head *hook_list, int event)
8902 {
8903         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
8904         struct sk_buff *skb;
8905         u16 flags = 0;
8906         int err;
8907
8908         if (!ctx->report &&
8909             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
8910                 return;
8911
8912         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
8913         if (skb == NULL)
8914                 goto err;
8915
8916         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
8917                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
8918
8919         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
8920                                             ctx->seq, event, flags,
8921                                             ctx->family, flowtable, hook_list);
8922         if (err < 0) {
8923                 kfree_skb(skb);
8924                 goto err;
8925         }
8926
8927         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
8928         return;
8929 err:
8930         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
8931 }
8932
8933 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
8934 {
8935         struct nft_hook *hook, *next;
8936
8937         flowtable->data.type->free(&flowtable->data);
8938         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8939                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8940                                             FLOW_BLOCK_UNBIND);
8941                 list_del_rcu(&hook->list);
8942                 kfree(hook);
8943         }
8944         kfree(flowtable->name);
8945         module_put(flowtable->data.type->owner);
8946         kfree(flowtable);
8947 }
8948
8949 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
8950                                    u32 portid, u32 seq)
8951 {
8952         struct nftables_pernet *nft_net = nft_pernet(net);
8953         struct nlmsghdr *nlh;
8954         char buf[TASK_COMM_LEN];
8955         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
8956
8957         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
8958                            NFNETLINK_V0, nft_base_seq(net));
8959         if (!nlh)
8960                 goto nla_put_failure;
8961
8962         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
8963             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
8964             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
8965                 goto nla_put_failure;
8966
8967         nlmsg_end(skb, nlh);
8968         return 0;
8969
8970 nla_put_failure:
8971         nlmsg_trim(skb, nlh);
8972         return -EMSGSIZE;
8973 }
8974
8975 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
8976                                 struct nft_flowtable *flowtable)
8977 {
8978         struct nft_hook *hook;
8979
8980         list_for_each_entry(hook, &flowtable->hook_list, list) {
8981                 if (hook->ops.dev != dev)
8982                         continue;
8983
8984                 /* flow_offload_netdev_event() cleans up entries for us. */
8985                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
8986                 list_del_rcu(&hook->list);
8987                 kfree_rcu(hook, rcu);
8988                 break;
8989         }
8990 }
8991
8992 static int nf_tables_flowtable_event(struct notifier_block *this,
8993                                      unsigned long event, void *ptr)
8994 {
8995         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
8996         struct nft_flowtable *flowtable;
8997         struct nftables_pernet *nft_net;
8998         struct nft_table *table;
8999         struct net *net;
9000
9001         if (event != NETDEV_UNREGISTER)
9002                 return 0;
9003
9004         net = dev_net(dev);
9005         nft_net = nft_pernet(net);
9006         mutex_lock(&nft_net->commit_mutex);
9007         list_for_each_entry(table, &nft_net->tables, list) {
9008                 list_for_each_entry(flowtable, &table->flowtables, list) {
9009                         nft_flowtable_event(event, dev, flowtable);
9010                 }
9011         }
9012         mutex_unlock(&nft_net->commit_mutex);
9013
9014         return NOTIFY_DONE;
9015 }
9016
9017 static struct notifier_block nf_tables_flowtable_notifier = {
9018         .notifier_call  = nf_tables_flowtable_event,
9019 };
9020
9021 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
9022                                  int event)
9023 {
9024         struct nlmsghdr *nlh = nlmsg_hdr(skb);
9025         struct sk_buff *skb2;
9026         int err;
9027
9028         if (!nlmsg_report(nlh) &&
9029             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
9030                 return;
9031
9032         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
9033         if (skb2 == NULL)
9034                 goto err;
9035
9036         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
9037                                       nlh->nlmsg_seq);
9038         if (err < 0) {
9039                 kfree_skb(skb2);
9040                 goto err;
9041         }
9042
9043         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
9044                        nlmsg_report(nlh), GFP_KERNEL);
9045         return;
9046 err:
9047         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
9048                           -ENOBUFS);
9049 }
9050
9051 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
9052                             const struct nlattr * const nla[])
9053 {
9054         struct sk_buff *skb2;
9055         int err;
9056
9057         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
9058         if (skb2 == NULL)
9059                 return -ENOMEM;
9060
9061         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
9062                                       info->nlh->nlmsg_seq);
9063         if (err < 0)
9064                 goto err_fill_gen_info;
9065
9066         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
9067
9068 err_fill_gen_info:
9069         kfree_skb(skb2);
9070         return err;
9071 }
9072
9073 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
9074         [NFT_MSG_NEWTABLE] = {
9075                 .call           = nf_tables_newtable,
9076                 .type           = NFNL_CB_BATCH,
9077                 .attr_count     = NFTA_TABLE_MAX,
9078                 .policy         = nft_table_policy,
9079         },
9080         [NFT_MSG_GETTABLE] = {
9081                 .call           = nf_tables_gettable,
9082                 .type           = NFNL_CB_RCU,
9083                 .attr_count     = NFTA_TABLE_MAX,
9084                 .policy         = nft_table_policy,
9085         },
9086         [NFT_MSG_DELTABLE] = {
9087                 .call           = nf_tables_deltable,
9088                 .type           = NFNL_CB_BATCH,
9089                 .attr_count     = NFTA_TABLE_MAX,
9090                 .policy         = nft_table_policy,
9091         },
9092         [NFT_MSG_DESTROYTABLE] = {
9093                 .call           = nf_tables_deltable,
9094                 .type           = NFNL_CB_BATCH,
9095                 .attr_count     = NFTA_TABLE_MAX,
9096                 .policy         = nft_table_policy,
9097         },
9098         [NFT_MSG_NEWCHAIN] = {
9099                 .call           = nf_tables_newchain,
9100                 .type           = NFNL_CB_BATCH,
9101                 .attr_count     = NFTA_CHAIN_MAX,
9102                 .policy         = nft_chain_policy,
9103         },
9104         [NFT_MSG_GETCHAIN] = {
9105                 .call           = nf_tables_getchain,
9106                 .type           = NFNL_CB_RCU,
9107                 .attr_count     = NFTA_CHAIN_MAX,
9108                 .policy         = nft_chain_policy,
9109         },
9110         [NFT_MSG_DELCHAIN] = {
9111                 .call           = nf_tables_delchain,
9112                 .type           = NFNL_CB_BATCH,
9113                 .attr_count     = NFTA_CHAIN_MAX,
9114                 .policy         = nft_chain_policy,
9115         },
9116         [NFT_MSG_DESTROYCHAIN] = {
9117                 .call           = nf_tables_delchain,
9118                 .type           = NFNL_CB_BATCH,
9119                 .attr_count     = NFTA_CHAIN_MAX,
9120                 .policy         = nft_chain_policy,
9121         },
9122         [NFT_MSG_NEWRULE] = {
9123                 .call           = nf_tables_newrule,
9124                 .type           = NFNL_CB_BATCH,
9125                 .attr_count     = NFTA_RULE_MAX,
9126                 .policy         = nft_rule_policy,
9127         },
9128         [NFT_MSG_GETRULE] = {
9129                 .call           = nf_tables_getrule,
9130                 .type           = NFNL_CB_RCU,
9131                 .attr_count     = NFTA_RULE_MAX,
9132                 .policy         = nft_rule_policy,
9133         },
9134         [NFT_MSG_GETRULE_RESET] = {
9135                 .call           = nf_tables_getrule_reset,
9136                 .type           = NFNL_CB_RCU,
9137                 .attr_count     = NFTA_RULE_MAX,
9138                 .policy         = nft_rule_policy,
9139         },
9140         [NFT_MSG_DELRULE] = {
9141                 .call           = nf_tables_delrule,
9142                 .type           = NFNL_CB_BATCH,
9143                 .attr_count     = NFTA_RULE_MAX,
9144                 .policy         = nft_rule_policy,
9145         },
9146         [NFT_MSG_DESTROYRULE] = {
9147                 .call           = nf_tables_delrule,
9148                 .type           = NFNL_CB_BATCH,
9149                 .attr_count     = NFTA_RULE_MAX,
9150                 .policy         = nft_rule_policy,
9151         },
9152         [NFT_MSG_NEWSET] = {
9153                 .call           = nf_tables_newset,
9154                 .type           = NFNL_CB_BATCH,
9155                 .attr_count     = NFTA_SET_MAX,
9156                 .policy         = nft_set_policy,
9157         },
9158         [NFT_MSG_GETSET] = {
9159                 .call           = nf_tables_getset,
9160                 .type           = NFNL_CB_RCU,
9161                 .attr_count     = NFTA_SET_MAX,
9162                 .policy         = nft_set_policy,
9163         },
9164         [NFT_MSG_DELSET] = {
9165                 .call           = nf_tables_delset,
9166                 .type           = NFNL_CB_BATCH,
9167                 .attr_count     = NFTA_SET_MAX,
9168                 .policy         = nft_set_policy,
9169         },
9170         [NFT_MSG_DESTROYSET] = {
9171                 .call           = nf_tables_delset,
9172                 .type           = NFNL_CB_BATCH,
9173                 .attr_count     = NFTA_SET_MAX,
9174                 .policy         = nft_set_policy,
9175         },
9176         [NFT_MSG_NEWSETELEM] = {
9177                 .call           = nf_tables_newsetelem,
9178                 .type           = NFNL_CB_BATCH,
9179                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9180                 .policy         = nft_set_elem_list_policy,
9181         },
9182         [NFT_MSG_GETSETELEM] = {
9183                 .call           = nf_tables_getsetelem,
9184                 .type           = NFNL_CB_RCU,
9185                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9186                 .policy         = nft_set_elem_list_policy,
9187         },
9188         [NFT_MSG_GETSETELEM_RESET] = {
9189                 .call           = nf_tables_getsetelem_reset,
9190                 .type           = NFNL_CB_RCU,
9191                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9192                 .policy         = nft_set_elem_list_policy,
9193         },
9194         [NFT_MSG_DELSETELEM] = {
9195                 .call           = nf_tables_delsetelem,
9196                 .type           = NFNL_CB_BATCH,
9197                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9198                 .policy         = nft_set_elem_list_policy,
9199         },
9200         [NFT_MSG_DESTROYSETELEM] = {
9201                 .call           = nf_tables_delsetelem,
9202                 .type           = NFNL_CB_BATCH,
9203                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9204                 .policy         = nft_set_elem_list_policy,
9205         },
9206         [NFT_MSG_GETGEN] = {
9207                 .call           = nf_tables_getgen,
9208                 .type           = NFNL_CB_RCU,
9209         },
9210         [NFT_MSG_NEWOBJ] = {
9211                 .call           = nf_tables_newobj,
9212                 .type           = NFNL_CB_BATCH,
9213                 .attr_count     = NFTA_OBJ_MAX,
9214                 .policy         = nft_obj_policy,
9215         },
9216         [NFT_MSG_GETOBJ] = {
9217                 .call           = nf_tables_getobj,
9218                 .type           = NFNL_CB_RCU,
9219                 .attr_count     = NFTA_OBJ_MAX,
9220                 .policy         = nft_obj_policy,
9221         },
9222         [NFT_MSG_DELOBJ] = {
9223                 .call           = nf_tables_delobj,
9224                 .type           = NFNL_CB_BATCH,
9225                 .attr_count     = NFTA_OBJ_MAX,
9226                 .policy         = nft_obj_policy,
9227         },
9228         [NFT_MSG_DESTROYOBJ] = {
9229                 .call           = nf_tables_delobj,
9230                 .type           = NFNL_CB_BATCH,
9231                 .attr_count     = NFTA_OBJ_MAX,
9232                 .policy         = nft_obj_policy,
9233         },
9234         [NFT_MSG_GETOBJ_RESET] = {
9235                 .call           = nf_tables_getobj,
9236                 .type           = NFNL_CB_RCU,
9237                 .attr_count     = NFTA_OBJ_MAX,
9238                 .policy         = nft_obj_policy,
9239         },
9240         [NFT_MSG_NEWFLOWTABLE] = {
9241                 .call           = nf_tables_newflowtable,
9242                 .type           = NFNL_CB_BATCH,
9243                 .attr_count     = NFTA_FLOWTABLE_MAX,
9244                 .policy         = nft_flowtable_policy,
9245         },
9246         [NFT_MSG_GETFLOWTABLE] = {
9247                 .call           = nf_tables_getflowtable,
9248                 .type           = NFNL_CB_RCU,
9249                 .attr_count     = NFTA_FLOWTABLE_MAX,
9250                 .policy         = nft_flowtable_policy,
9251         },
9252         [NFT_MSG_DELFLOWTABLE] = {
9253                 .call           = nf_tables_delflowtable,
9254                 .type           = NFNL_CB_BATCH,
9255                 .attr_count     = NFTA_FLOWTABLE_MAX,
9256                 .policy         = nft_flowtable_policy,
9257         },
9258         [NFT_MSG_DESTROYFLOWTABLE] = {
9259                 .call           = nf_tables_delflowtable,
9260                 .type           = NFNL_CB_BATCH,
9261                 .attr_count     = NFTA_FLOWTABLE_MAX,
9262                 .policy         = nft_flowtable_policy,
9263         },
9264 };
9265
9266 static int nf_tables_validate(struct net *net)
9267 {
9268         struct nftables_pernet *nft_net = nft_pernet(net);
9269         struct nft_table *table;
9270
9271         list_for_each_entry(table, &nft_net->tables, list) {
9272                 switch (table->validate_state) {
9273                 case NFT_VALIDATE_SKIP:
9274                         continue;
9275                 case NFT_VALIDATE_NEED:
9276                         nft_validate_state_update(table, NFT_VALIDATE_DO);
9277                         fallthrough;
9278                 case NFT_VALIDATE_DO:
9279                         if (nft_table_validate(net, table) < 0)
9280                                 return -EAGAIN;
9281
9282                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
9283                         break;
9284                 }
9285         }
9286
9287         return 0;
9288 }
9289
9290 /* a drop policy has to be deferred until all rules have been activated,
9291  * otherwise a large ruleset that contains a drop-policy base chain will
9292  * cause all packets to get dropped until the full transaction has been
9293  * processed.
9294  *
9295  * We defer the drop policy until the transaction has been finalized.
9296  */
9297 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
9298 {
9299         struct nft_base_chain *basechain;
9300
9301         if (nft_trans_chain_policy(trans) != NF_DROP)
9302                 return;
9303
9304         if (!nft_is_base_chain(trans->ctx.chain))
9305                 return;
9306
9307         basechain = nft_base_chain(trans->ctx.chain);
9308         basechain->policy = NF_DROP;
9309 }
9310
9311 static void nft_chain_commit_update(struct nft_trans *trans)
9312 {
9313         struct nft_base_chain *basechain;
9314
9315         if (nft_trans_chain_name(trans)) {
9316                 rhltable_remove(&trans->ctx.table->chains_ht,
9317                                 &trans->ctx.chain->rhlhead,
9318                                 nft_chain_ht_params);
9319                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
9320                 rhltable_insert_key(&trans->ctx.table->chains_ht,
9321                                     trans->ctx.chain->name,
9322                                     &trans->ctx.chain->rhlhead,
9323                                     nft_chain_ht_params);
9324         }
9325
9326         if (!nft_is_base_chain(trans->ctx.chain))
9327                 return;
9328
9329         nft_chain_stats_replace(trans);
9330
9331         basechain = nft_base_chain(trans->ctx.chain);
9332
9333         switch (nft_trans_chain_policy(trans)) {
9334         case NF_DROP:
9335         case NF_ACCEPT:
9336                 basechain->policy = nft_trans_chain_policy(trans);
9337                 break;
9338         }
9339 }
9340
9341 static void nft_obj_commit_update(struct nft_trans *trans)
9342 {
9343         struct nft_object *newobj;
9344         struct nft_object *obj;
9345
9346         obj = nft_trans_obj(trans);
9347         newobj = nft_trans_obj_newobj(trans);
9348
9349         if (obj->ops->update)
9350                 obj->ops->update(obj, newobj);
9351
9352         nft_obj_destroy(&trans->ctx, newobj);
9353 }
9354
9355 static void nft_commit_release(struct nft_trans *trans)
9356 {
9357         switch (trans->msg_type) {
9358         case NFT_MSG_DELTABLE:
9359         case NFT_MSG_DESTROYTABLE:
9360                 nf_tables_table_destroy(&trans->ctx);
9361                 break;
9362         case NFT_MSG_NEWCHAIN:
9363                 free_percpu(nft_trans_chain_stats(trans));
9364                 kfree(nft_trans_chain_name(trans));
9365                 break;
9366         case NFT_MSG_DELCHAIN:
9367         case NFT_MSG_DESTROYCHAIN:
9368                 if (nft_trans_chain_update(trans))
9369                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9370                 else
9371                         nf_tables_chain_destroy(&trans->ctx);
9372                 break;
9373         case NFT_MSG_DELRULE:
9374         case NFT_MSG_DESTROYRULE:
9375                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9376                 break;
9377         case NFT_MSG_DELSET:
9378         case NFT_MSG_DESTROYSET:
9379                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9380                 break;
9381         case NFT_MSG_DELSETELEM:
9382         case NFT_MSG_DESTROYSETELEM:
9383                 nf_tables_set_elem_destroy(&trans->ctx,
9384                                            nft_trans_elem_set(trans),
9385                                            nft_trans_elem_priv(trans));
9386                 break;
9387         case NFT_MSG_DELOBJ:
9388         case NFT_MSG_DESTROYOBJ:
9389                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9390                 break;
9391         case NFT_MSG_DELFLOWTABLE:
9392         case NFT_MSG_DESTROYFLOWTABLE:
9393                 if (nft_trans_flowtable_update(trans))
9394                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9395                 else
9396                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9397                 break;
9398         }
9399
9400         if (trans->put_net)
9401                 put_net(trans->ctx.net);
9402
9403         kfree(trans);
9404 }
9405
9406 static void nf_tables_trans_destroy_work(struct work_struct *w)
9407 {
9408         struct nft_trans *trans, *next;
9409         LIST_HEAD(head);
9410
9411         spin_lock(&nf_tables_destroy_list_lock);
9412         list_splice_init(&nf_tables_destroy_list, &head);
9413         spin_unlock(&nf_tables_destroy_list_lock);
9414
9415         if (list_empty(&head))
9416                 return;
9417
9418         synchronize_rcu();
9419
9420         list_for_each_entry_safe(trans, next, &head, list) {
9421                 nft_trans_list_del(trans);
9422                 nft_commit_release(trans);
9423         }
9424 }
9425
9426 void nf_tables_trans_destroy_flush_work(void)
9427 {
9428         flush_work(&trans_destroy_work);
9429 }
9430 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
9431
9432 static bool nft_expr_reduce(struct nft_regs_track *track,
9433                             const struct nft_expr *expr)
9434 {
9435         return false;
9436 }
9437
9438 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
9439 {
9440         const struct nft_expr *expr, *last;
9441         struct nft_regs_track track = {};
9442         unsigned int size, data_size;
9443         void *data, *data_boundary;
9444         struct nft_rule_dp *prule;
9445         struct nft_rule *rule;
9446
9447         /* already handled or inactive chain? */
9448         if (chain->blob_next || !nft_is_active_next(net, chain))
9449                 return 0;
9450
9451         data_size = 0;
9452         list_for_each_entry(rule, &chain->rules, list) {
9453                 if (nft_is_active_next(net, rule)) {
9454                         data_size += sizeof(*prule) + rule->dlen;
9455                         if (data_size > INT_MAX)
9456                                 return -ENOMEM;
9457                 }
9458         }
9459
9460         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
9461         if (!chain->blob_next)
9462                 return -ENOMEM;
9463
9464         data = (void *)chain->blob_next->data;
9465         data_boundary = data + data_size;
9466         size = 0;
9467
9468         list_for_each_entry(rule, &chain->rules, list) {
9469                 if (!nft_is_active_next(net, rule))
9470                         continue;
9471
9472                 prule = (struct nft_rule_dp *)data;
9473                 data += offsetof(struct nft_rule_dp, data);
9474                 if (WARN_ON_ONCE(data > data_boundary))
9475                         return -ENOMEM;
9476
9477                 size = 0;
9478                 track.last = nft_expr_last(rule);
9479                 nft_rule_for_each_expr(expr, last, rule) {
9480                         track.cur = expr;
9481
9482                         if (nft_expr_reduce(&track, expr)) {
9483                                 expr = track.cur;
9484                                 continue;
9485                         }
9486
9487                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
9488                                 return -ENOMEM;
9489
9490                         memcpy(data + size, expr, expr->ops->size);
9491                         size += expr->ops->size;
9492                 }
9493                 if (WARN_ON_ONCE(size >= 1 << 12))
9494                         return -ENOMEM;
9495
9496                 prule->handle = rule->handle;
9497                 prule->dlen = size;
9498                 prule->is_last = 0;
9499
9500                 data += size;
9501                 size = 0;
9502                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
9503         }
9504
9505         if (WARN_ON_ONCE(data > data_boundary))
9506                 return -ENOMEM;
9507
9508         prule = (struct nft_rule_dp *)data;
9509         nft_last_rule(chain, prule);
9510
9511         return 0;
9512 }
9513
9514 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
9515 {
9516         struct nftables_pernet *nft_net = nft_pernet(net);
9517         struct nft_trans *trans, *next;
9518
9519         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9520                 struct nft_chain *chain = trans->ctx.chain;
9521
9522                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9523                     trans->msg_type == NFT_MSG_DELRULE) {
9524                         kvfree(chain->blob_next);
9525                         chain->blob_next = NULL;
9526                 }
9527         }
9528 }
9529
9530 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
9531 {
9532         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
9533
9534         kvfree(l->blob);
9535 }
9536
9537 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
9538 {
9539         struct nft_rule_dp_last *last;
9540
9541         /* last rule trailer is after end marker */
9542         last = (void *)blob + sizeof(*blob) + blob->size;
9543         last->blob = blob;
9544
9545         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
9546 }
9547
9548 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
9549 {
9550         struct nft_rule_blob *g0, *g1;
9551         bool next_genbit;
9552
9553         next_genbit = nft_gencursor_next(net);
9554
9555         g0 = rcu_dereference_protected(chain->blob_gen_0,
9556                                        lockdep_commit_lock_is_held(net));
9557         g1 = rcu_dereference_protected(chain->blob_gen_1,
9558                                        lockdep_commit_lock_is_held(net));
9559
9560         /* No changes to this chain? */
9561         if (chain->blob_next == NULL) {
9562                 /* chain had no change in last or next generation */
9563                 if (g0 == g1)
9564                         return;
9565                 /*
9566                  * chain had no change in this generation; make sure next
9567                  * one uses same rules as current generation.
9568                  */
9569                 if (next_genbit) {
9570                         rcu_assign_pointer(chain->blob_gen_1, g0);
9571                         nf_tables_commit_chain_free_rules_old(g1);
9572                 } else {
9573                         rcu_assign_pointer(chain->blob_gen_0, g1);
9574                         nf_tables_commit_chain_free_rules_old(g0);
9575                 }
9576
9577                 return;
9578         }
9579
9580         if (next_genbit)
9581                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
9582         else
9583                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
9584
9585         chain->blob_next = NULL;
9586
9587         if (g0 == g1)
9588                 return;
9589
9590         if (next_genbit)
9591                 nf_tables_commit_chain_free_rules_old(g1);
9592         else
9593                 nf_tables_commit_chain_free_rules_old(g0);
9594 }
9595
9596 static void nft_obj_del(struct nft_object *obj)
9597 {
9598         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
9599         list_del_rcu(&obj->list);
9600 }
9601
9602 void nft_chain_del(struct nft_chain *chain)
9603 {
9604         struct nft_table *table = chain->table;
9605
9606         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
9607                                      nft_chain_ht_params));
9608         list_del_rcu(&chain->list);
9609 }
9610
9611 static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
9612                                         struct nft_trans_gc *trans)
9613 {
9614         struct nft_elem_priv **priv = trans->priv;
9615         unsigned int i;
9616
9617         for (i = 0; i < trans->count; i++) {
9618                 nft_setelem_data_deactivate(ctx->net, trans->set, priv[i]);
9619                 nft_setelem_remove(ctx->net, trans->set, priv[i]);
9620         }
9621 }
9622
9623 void nft_trans_gc_destroy(struct nft_trans_gc *trans)
9624 {
9625         nft_set_put(trans->set);
9626         put_net(trans->net);
9627         kfree(trans);
9628 }
9629
9630 static void nft_trans_gc_trans_free(struct rcu_head *rcu)
9631 {
9632         struct nft_elem_priv *elem_priv;
9633         struct nft_trans_gc *trans;
9634         struct nft_ctx ctx = {};
9635         unsigned int i;
9636
9637         trans = container_of(rcu, struct nft_trans_gc, rcu);
9638         ctx.net = read_pnet(&trans->set->net);
9639
9640         for (i = 0; i < trans->count; i++) {
9641                 elem_priv = trans->priv[i];
9642                 if (!nft_setelem_is_catchall(trans->set, elem_priv))
9643                         atomic_dec(&trans->set->nelems);
9644
9645                 nf_tables_set_elem_destroy(&ctx, trans->set, elem_priv);
9646         }
9647
9648         nft_trans_gc_destroy(trans);
9649 }
9650
9651 static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
9652 {
9653         struct nftables_pernet *nft_net;
9654         struct nft_ctx ctx = {};
9655
9656         nft_net = nft_pernet(trans->net);
9657
9658         mutex_lock(&nft_net->commit_mutex);
9659
9660         /* Check for race with transaction, otherwise this batch refers to
9661          * stale objects that might not be there anymore. Skip transaction if
9662          * set has been destroyed from control plane transaction in case gc
9663          * worker loses race.
9664          */
9665         if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
9666                 mutex_unlock(&nft_net->commit_mutex);
9667                 return false;
9668         }
9669
9670         ctx.net = trans->net;
9671         ctx.table = trans->set->table;
9672
9673         nft_trans_gc_setelem_remove(&ctx, trans);
9674         mutex_unlock(&nft_net->commit_mutex);
9675
9676         return true;
9677 }
9678
9679 static void nft_trans_gc_work(struct work_struct *work)
9680 {
9681         struct nft_trans_gc *trans, *next;
9682         LIST_HEAD(trans_gc_list);
9683
9684         spin_lock(&nf_tables_gc_list_lock);
9685         list_splice_init(&nf_tables_gc_list, &trans_gc_list);
9686         spin_unlock(&nf_tables_gc_list_lock);
9687
9688         list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
9689                 list_del(&trans->list);
9690                 if (!nft_trans_gc_work_done(trans)) {
9691                         nft_trans_gc_destroy(trans);
9692                         continue;
9693                 }
9694                 call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9695         }
9696 }
9697
9698 struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
9699                                         unsigned int gc_seq, gfp_t gfp)
9700 {
9701         struct net *net = read_pnet(&set->net);
9702         struct nft_trans_gc *trans;
9703
9704         trans = kzalloc(sizeof(*trans), gfp);
9705         if (!trans)
9706                 return NULL;
9707
9708         trans->net = maybe_get_net(net);
9709         if (!trans->net) {
9710                 kfree(trans);
9711                 return NULL;
9712         }
9713
9714         refcount_inc(&set->refs);
9715         trans->set = set;
9716         trans->seq = gc_seq;
9717
9718         return trans;
9719 }
9720
9721 void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
9722 {
9723         trans->priv[trans->count++] = priv;
9724 }
9725
9726 static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
9727 {
9728         spin_lock(&nf_tables_gc_list_lock);
9729         list_add_tail(&trans->list, &nf_tables_gc_list);
9730         spin_unlock(&nf_tables_gc_list_lock);
9731
9732         schedule_work(&trans_gc_work);
9733 }
9734
9735 static int nft_trans_gc_space(struct nft_trans_gc *trans)
9736 {
9737         return NFT_TRANS_GC_BATCHCOUNT - trans->count;
9738 }
9739
9740 struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
9741                                               unsigned int gc_seq, gfp_t gfp)
9742 {
9743         struct nft_set *set;
9744
9745         if (nft_trans_gc_space(gc))
9746                 return gc;
9747
9748         set = gc->set;
9749         nft_trans_gc_queue_work(gc);
9750
9751         return nft_trans_gc_alloc(set, gc_seq, gfp);
9752 }
9753
9754 void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
9755 {
9756         if (trans->count == 0) {
9757                 nft_trans_gc_destroy(trans);
9758                 return;
9759         }
9760
9761         nft_trans_gc_queue_work(trans);
9762 }
9763
9764 struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
9765 {
9766         struct nft_set *set;
9767
9768         if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
9769                 return NULL;
9770
9771         if (nft_trans_gc_space(gc))
9772                 return gc;
9773
9774         set = gc->set;
9775         call_rcu(&gc->rcu, nft_trans_gc_trans_free);
9776
9777         return nft_trans_gc_alloc(set, 0, gfp);
9778 }
9779
9780 void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
9781 {
9782         WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
9783
9784         if (trans->count == 0) {
9785                 nft_trans_gc_destroy(trans);
9786                 return;
9787         }
9788
9789         call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9790 }
9791
9792 struct nft_trans_gc *nft_trans_gc_catchall_async(struct nft_trans_gc *gc,
9793                                                  unsigned int gc_seq)
9794 {
9795         struct nft_set_elem_catchall *catchall;
9796         const struct nft_set *set = gc->set;
9797         struct nft_set_ext *ext;
9798
9799         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9800                 ext = nft_set_elem_ext(set, catchall->elem);
9801
9802                 if (!nft_set_elem_expired(ext))
9803                         continue;
9804                 if (nft_set_elem_is_dead(ext))
9805                         goto dead_elem;
9806
9807                 nft_set_elem_dead(ext);
9808 dead_elem:
9809                 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
9810                 if (!gc)
9811                         return NULL;
9812
9813                 nft_trans_gc_elem_add(gc, catchall->elem);
9814         }
9815
9816         return gc;
9817 }
9818
9819 struct nft_trans_gc *nft_trans_gc_catchall_sync(struct nft_trans_gc *gc)
9820 {
9821         struct nft_set_elem_catchall *catchall, *next;
9822         const struct nft_set *set = gc->set;
9823         struct nft_elem_priv *elem_priv;
9824         struct nft_set_ext *ext;
9825
9826         WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net));
9827
9828         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
9829                 ext = nft_set_elem_ext(set, catchall->elem);
9830
9831                 if (!nft_set_elem_expired(ext))
9832                         continue;
9833
9834                 gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL);
9835                 if (!gc)
9836                         return NULL;
9837
9838                 elem_priv = catchall->elem;
9839                 nft_setelem_data_deactivate(gc->net, gc->set, elem_priv);
9840                 nft_setelem_catchall_destroy(catchall);
9841                 nft_trans_gc_elem_add(gc, elem_priv);
9842         }
9843
9844         return gc;
9845 }
9846
9847 static void nf_tables_module_autoload_cleanup(struct net *net)
9848 {
9849         struct nftables_pernet *nft_net = nft_pernet(net);
9850         struct nft_module_request *req, *next;
9851
9852         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
9853         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
9854                 WARN_ON_ONCE(!req->done);
9855                 list_del(&req->list);
9856                 kfree(req);
9857         }
9858 }
9859
9860 static void nf_tables_commit_release(struct net *net)
9861 {
9862         struct nftables_pernet *nft_net = nft_pernet(net);
9863         struct nft_trans *trans;
9864
9865         /* all side effects have to be made visible.
9866          * For example, if a chain named 'foo' has been deleted, a
9867          * new transaction must not find it anymore.
9868          *
9869          * Memory reclaim happens asynchronously from work queue
9870          * to prevent expensive synchronize_rcu() in commit phase.
9871          */
9872         if (list_empty(&nft_net->commit_list)) {
9873                 nf_tables_module_autoload_cleanup(net);
9874                 mutex_unlock(&nft_net->commit_mutex);
9875                 return;
9876         }
9877
9878         trans = list_last_entry(&nft_net->commit_list,
9879                                 struct nft_trans, list);
9880         get_net(trans->ctx.net);
9881         WARN_ON_ONCE(trans->put_net);
9882
9883         trans->put_net = true;
9884         spin_lock(&nf_tables_destroy_list_lock);
9885         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
9886         spin_unlock(&nf_tables_destroy_list_lock);
9887
9888         nf_tables_module_autoload_cleanup(net);
9889         schedule_work(&trans_destroy_work);
9890
9891         mutex_unlock(&nft_net->commit_mutex);
9892 }
9893
9894 static void nft_commit_notify(struct net *net, u32 portid)
9895 {
9896         struct nftables_pernet *nft_net = nft_pernet(net);
9897         struct sk_buff *batch_skb = NULL, *nskb, *skb;
9898         unsigned char *data;
9899         int len;
9900
9901         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
9902                 if (!batch_skb) {
9903 new_batch:
9904                         batch_skb = skb;
9905                         len = NLMSG_GOODSIZE - skb->len;
9906                         list_del(&skb->list);
9907                         continue;
9908                 }
9909                 len -= skb->len;
9910                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
9911                         data = skb_put(batch_skb, skb->len);
9912                         memcpy(data, skb->data, skb->len);
9913                         list_del(&skb->list);
9914                         kfree_skb(skb);
9915                         continue;
9916                 }
9917                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9918                                NFT_CB(batch_skb).report, GFP_KERNEL);
9919                 goto new_batch;
9920         }
9921
9922         if (batch_skb) {
9923                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
9924                                NFT_CB(batch_skb).report, GFP_KERNEL);
9925         }
9926
9927         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
9928 }
9929
9930 static int nf_tables_commit_audit_alloc(struct list_head *adl,
9931                                         struct nft_table *table)
9932 {
9933         struct nft_audit_data *adp;
9934
9935         list_for_each_entry(adp, adl, list) {
9936                 if (adp->table == table)
9937                         return 0;
9938         }
9939         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
9940         if (!adp)
9941                 return -ENOMEM;
9942         adp->table = table;
9943         list_add(&adp->list, adl);
9944         return 0;
9945 }
9946
9947 static void nf_tables_commit_audit_free(struct list_head *adl)
9948 {
9949         struct nft_audit_data *adp, *adn;
9950
9951         list_for_each_entry_safe(adp, adn, adl, list) {
9952                 list_del(&adp->list);
9953                 kfree(adp);
9954         }
9955 }
9956
9957 static void nf_tables_commit_audit_collect(struct list_head *adl,
9958                                            struct nft_table *table, u32 op)
9959 {
9960         struct nft_audit_data *adp;
9961
9962         list_for_each_entry(adp, adl, list) {
9963                 if (adp->table == table)
9964                         goto found;
9965         }
9966         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
9967         return;
9968 found:
9969         adp->entries++;
9970         if (!adp->op || adp->op > op)
9971                 adp->op = op;
9972 }
9973
9974 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
9975
9976 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
9977 {
9978         struct nft_audit_data *adp, *adn;
9979         char aubuf[AUNFTABLENAMELEN];
9980
9981         list_for_each_entry_safe(adp, adn, adl, list) {
9982                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
9983                          generation);
9984                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
9985                                 nft2audit_op[adp->op], GFP_KERNEL);
9986                 list_del(&adp->list);
9987                 kfree(adp);
9988         }
9989 }
9990
9991 static void nft_set_commit_update(struct list_head *set_update_list)
9992 {
9993         struct nft_set *set, *next;
9994
9995         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
9996                 list_del_init(&set->pending_update);
9997
9998                 if (!set->ops->commit || set->dead)
9999                         continue;
10000
10001                 set->ops->commit(set);
10002         }
10003 }
10004
10005 static unsigned int nft_gc_seq_begin(struct nftables_pernet *nft_net)
10006 {
10007         unsigned int gc_seq;
10008
10009         /* Bump gc counter, it becomes odd, this is the busy mark. */
10010         gc_seq = READ_ONCE(nft_net->gc_seq);
10011         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10012
10013         return gc_seq;
10014 }
10015
10016 static void nft_gc_seq_end(struct nftables_pernet *nft_net, unsigned int gc_seq)
10017 {
10018         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10019 }
10020
10021 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
10022 {
10023         struct nftables_pernet *nft_net = nft_pernet(net);
10024         struct nft_trans *trans, *next;
10025         unsigned int base_seq, gc_seq;
10026         LIST_HEAD(set_update_list);
10027         struct nft_trans_elem *te;
10028         struct nft_chain *chain;
10029         struct nft_table *table;
10030         LIST_HEAD(adl);
10031         int err;
10032
10033         if (list_empty(&nft_net->commit_list)) {
10034                 mutex_unlock(&nft_net->commit_mutex);
10035                 return 0;
10036         }
10037
10038         list_for_each_entry(trans, &nft_net->binding_list, binding_list) {
10039                 switch (trans->msg_type) {
10040                 case NFT_MSG_NEWSET:
10041                         if (!nft_trans_set_update(trans) &&
10042                             nft_set_is_anonymous(nft_trans_set(trans)) &&
10043                             !nft_trans_set_bound(trans)) {
10044                                 pr_warn_once("nftables ruleset with unbound set\n");
10045                                 return -EINVAL;
10046                         }
10047                         break;
10048                 case NFT_MSG_NEWCHAIN:
10049                         if (!nft_trans_chain_update(trans) &&
10050                             nft_chain_binding(nft_trans_chain(trans)) &&
10051                             !nft_trans_chain_bound(trans)) {
10052                                 pr_warn_once("nftables ruleset with unbound chain\n");
10053                                 return -EINVAL;
10054                         }
10055                         break;
10056                 }
10057         }
10058
10059         /* 0. Validate ruleset, otherwise roll back for error reporting. */
10060         if (nf_tables_validate(net) < 0) {
10061                 nft_net->validate_state = NFT_VALIDATE_DO;
10062                 return -EAGAIN;
10063         }
10064
10065         err = nft_flow_rule_offload_commit(net);
10066         if (err < 0)
10067                 return err;
10068
10069         /* 1.  Allocate space for next generation rules_gen_X[] */
10070         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10071                 int ret;
10072
10073                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
10074                 if (ret) {
10075                         nf_tables_commit_chain_prepare_cancel(net);
10076                         nf_tables_commit_audit_free(&adl);
10077                         return ret;
10078                 }
10079                 if (trans->msg_type == NFT_MSG_NEWRULE ||
10080                     trans->msg_type == NFT_MSG_DELRULE) {
10081                         chain = trans->ctx.chain;
10082
10083                         ret = nf_tables_commit_chain_prepare(net, chain);
10084                         if (ret < 0) {
10085                                 nf_tables_commit_chain_prepare_cancel(net);
10086                                 nf_tables_commit_audit_free(&adl);
10087                                 return ret;
10088                         }
10089                 }
10090         }
10091
10092         /* step 2.  Make rules_gen_X visible to packet path */
10093         list_for_each_entry(table, &nft_net->tables, list) {
10094                 list_for_each_entry(chain, &table->chains, list)
10095                         nf_tables_commit_chain(net, chain);
10096         }
10097
10098         /*
10099          * Bump generation counter, invalidate any dump in progress.
10100          * Cannot fail after this point.
10101          */
10102         base_seq = READ_ONCE(nft_net->base_seq);
10103         while (++base_seq == 0)
10104                 ;
10105
10106         WRITE_ONCE(nft_net->base_seq, base_seq);
10107
10108         gc_seq = nft_gc_seq_begin(nft_net);
10109
10110         /* step 3. Start new generation, rules_gen_X now in use. */
10111         net->nft.gencursor = nft_gencursor_next(net);
10112
10113         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10114                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
10115                                                trans->msg_type);
10116                 switch (trans->msg_type) {
10117                 case NFT_MSG_NEWTABLE:
10118                         if (nft_trans_table_update(trans)) {
10119                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10120                                         nft_trans_destroy(trans);
10121                                         break;
10122                                 }
10123                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
10124                                         nf_tables_table_disable(net, trans->ctx.table);
10125
10126                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10127                         } else {
10128                                 nft_clear(net, trans->ctx.table);
10129                         }
10130                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
10131                         nft_trans_destroy(trans);
10132                         break;
10133                 case NFT_MSG_DELTABLE:
10134                 case NFT_MSG_DESTROYTABLE:
10135                         list_del_rcu(&trans->ctx.table->list);
10136                         nf_tables_table_notify(&trans->ctx, trans->msg_type);
10137                         break;
10138                 case NFT_MSG_NEWCHAIN:
10139                         if (nft_trans_chain_update(trans)) {
10140                                 nft_chain_commit_update(trans);
10141                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN,
10142                                                        &nft_trans_chain_hooks(trans));
10143                                 list_splice(&nft_trans_chain_hooks(trans),
10144                                             &nft_trans_basechain(trans)->hook_list);
10145                                 /* trans destroyed after rcu grace period */
10146                         } else {
10147                                 nft_chain_commit_drop_policy(trans);
10148                                 nft_clear(net, trans->ctx.chain);
10149                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL);
10150                                 nft_trans_destroy(trans);
10151                         }
10152                         break;
10153                 case NFT_MSG_DELCHAIN:
10154                 case NFT_MSG_DESTROYCHAIN:
10155                         if (nft_trans_chain_update(trans)) {
10156                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
10157                                                        &nft_trans_chain_hooks(trans));
10158                                 nft_netdev_unregister_hooks(net,
10159                                                             &nft_trans_chain_hooks(trans),
10160                                                             true);
10161                         } else {
10162                                 nft_chain_del(trans->ctx.chain);
10163                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
10164                                                        NULL);
10165                                 nf_tables_unregister_hook(trans->ctx.net,
10166                                                           trans->ctx.table,
10167                                                           trans->ctx.chain);
10168                         }
10169                         break;
10170                 case NFT_MSG_NEWRULE:
10171                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10172                         nf_tables_rule_notify(&trans->ctx,
10173                                               nft_trans_rule(trans),
10174                                               NFT_MSG_NEWRULE);
10175                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10176                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10177
10178                         nft_trans_destroy(trans);
10179                         break;
10180                 case NFT_MSG_DELRULE:
10181                 case NFT_MSG_DESTROYRULE:
10182                         list_del_rcu(&nft_trans_rule(trans)->list);
10183                         nf_tables_rule_notify(&trans->ctx,
10184                                               nft_trans_rule(trans),
10185                                               trans->msg_type);
10186                         nft_rule_expr_deactivate(&trans->ctx,
10187                                                  nft_trans_rule(trans),
10188                                                  NFT_TRANS_COMMIT);
10189
10190                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10191                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10192                         break;
10193                 case NFT_MSG_NEWSET:
10194                         if (nft_trans_set_update(trans)) {
10195                                 struct nft_set *set = nft_trans_set(trans);
10196
10197                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
10198                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
10199
10200                                 if (nft_trans_set_size(trans))
10201                                         WRITE_ONCE(set->size, nft_trans_set_size(trans));
10202                         } else {
10203                                 nft_clear(net, nft_trans_set(trans));
10204                                 /* This avoids hitting -EBUSY when deleting the table
10205                                  * from the transaction.
10206                                  */
10207                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
10208                                     !list_empty(&nft_trans_set(trans)->bindings))
10209                                         nft_use_dec(&trans->ctx.table->use);
10210                         }
10211                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10212                                              NFT_MSG_NEWSET, GFP_KERNEL);
10213                         nft_trans_destroy(trans);
10214                         break;
10215                 case NFT_MSG_DELSET:
10216                 case NFT_MSG_DESTROYSET:
10217                         nft_trans_set(trans)->dead = 1;
10218                         list_del_rcu(&nft_trans_set(trans)->list);
10219                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10220                                              trans->msg_type, GFP_KERNEL);
10221                         break;
10222                 case NFT_MSG_NEWSETELEM:
10223                         te = (struct nft_trans_elem *)trans->data;
10224
10225                         nft_setelem_activate(net, te->set, te->elem_priv);
10226                         nf_tables_setelem_notify(&trans->ctx, te->set,
10227                                                  te->elem_priv,
10228                                                  NFT_MSG_NEWSETELEM);
10229                         if (te->set->ops->commit &&
10230                             list_empty(&te->set->pending_update)) {
10231                                 list_add_tail(&te->set->pending_update,
10232                                               &set_update_list);
10233                         }
10234                         nft_trans_destroy(trans);
10235                         break;
10236                 case NFT_MSG_DELSETELEM:
10237                 case NFT_MSG_DESTROYSETELEM:
10238                         te = (struct nft_trans_elem *)trans->data;
10239
10240                         nf_tables_setelem_notify(&trans->ctx, te->set,
10241                                                  te->elem_priv,
10242                                                  trans->msg_type);
10243                         nft_setelem_remove(net, te->set, te->elem_priv);
10244                         if (!nft_setelem_is_catchall(te->set, te->elem_priv)) {
10245                                 atomic_dec(&te->set->nelems);
10246                                 te->set->ndeact--;
10247                         }
10248                         if (te->set->ops->commit &&
10249                             list_empty(&te->set->pending_update)) {
10250                                 list_add_tail(&te->set->pending_update,
10251                                               &set_update_list);
10252                         }
10253                         break;
10254                 case NFT_MSG_NEWOBJ:
10255                         if (nft_trans_obj_update(trans)) {
10256                                 nft_obj_commit_update(trans);
10257                                 nf_tables_obj_notify(&trans->ctx,
10258                                                      nft_trans_obj(trans),
10259                                                      NFT_MSG_NEWOBJ);
10260                         } else {
10261                                 nft_clear(net, nft_trans_obj(trans));
10262                                 nf_tables_obj_notify(&trans->ctx,
10263                                                      nft_trans_obj(trans),
10264                                                      NFT_MSG_NEWOBJ);
10265                                 nft_trans_destroy(trans);
10266                         }
10267                         break;
10268                 case NFT_MSG_DELOBJ:
10269                 case NFT_MSG_DESTROYOBJ:
10270                         nft_obj_del(nft_trans_obj(trans));
10271                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
10272                                              trans->msg_type);
10273                         break;
10274                 case NFT_MSG_NEWFLOWTABLE:
10275                         if (nft_trans_flowtable_update(trans)) {
10276                                 nft_trans_flowtable(trans)->data.flags =
10277                                         nft_trans_flowtable_flags(trans);
10278                                 nf_tables_flowtable_notify(&trans->ctx,
10279                                                            nft_trans_flowtable(trans),
10280                                                            &nft_trans_flowtable_hooks(trans),
10281                                                            NFT_MSG_NEWFLOWTABLE);
10282                                 list_splice(&nft_trans_flowtable_hooks(trans),
10283                                             &nft_trans_flowtable(trans)->hook_list);
10284                         } else {
10285                                 nft_clear(net, nft_trans_flowtable(trans));
10286                                 nf_tables_flowtable_notify(&trans->ctx,
10287                                                            nft_trans_flowtable(trans),
10288                                                            NULL,
10289                                                            NFT_MSG_NEWFLOWTABLE);
10290                         }
10291                         nft_trans_destroy(trans);
10292                         break;
10293                 case NFT_MSG_DELFLOWTABLE:
10294                 case NFT_MSG_DESTROYFLOWTABLE:
10295                         if (nft_trans_flowtable_update(trans)) {
10296                                 nf_tables_flowtable_notify(&trans->ctx,
10297                                                            nft_trans_flowtable(trans),
10298                                                            &nft_trans_flowtable_hooks(trans),
10299                                                            trans->msg_type);
10300                                 nft_unregister_flowtable_net_hooks(net,
10301                                                                    &nft_trans_flowtable_hooks(trans));
10302                         } else {
10303                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10304                                 nf_tables_flowtable_notify(&trans->ctx,
10305                                                            nft_trans_flowtable(trans),
10306                                                            NULL,
10307                                                            trans->msg_type);
10308                                 nft_unregister_flowtable_net_hooks(net,
10309                                                 &nft_trans_flowtable(trans)->hook_list);
10310                         }
10311                         break;
10312                 }
10313         }
10314
10315         nft_set_commit_update(&set_update_list);
10316
10317         nft_commit_notify(net, NETLINK_CB(skb).portid);
10318         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
10319         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
10320
10321         nft_gc_seq_end(nft_net, gc_seq);
10322         nft_net->validate_state = NFT_VALIDATE_SKIP;
10323         nf_tables_commit_release(net);
10324
10325         return 0;
10326 }
10327
10328 static void nf_tables_module_autoload(struct net *net)
10329 {
10330         struct nftables_pernet *nft_net = nft_pernet(net);
10331         struct nft_module_request *req, *next;
10332         LIST_HEAD(module_list);
10333
10334         list_splice_init(&nft_net->module_list, &module_list);
10335         mutex_unlock(&nft_net->commit_mutex);
10336         list_for_each_entry_safe(req, next, &module_list, list) {
10337                 request_module("%s", req->module);
10338                 req->done = true;
10339         }
10340         mutex_lock(&nft_net->commit_mutex);
10341         list_splice(&module_list, &nft_net->module_list);
10342 }
10343
10344 static void nf_tables_abort_release(struct nft_trans *trans)
10345 {
10346         switch (trans->msg_type) {
10347         case NFT_MSG_NEWTABLE:
10348                 nf_tables_table_destroy(&trans->ctx);
10349                 break;
10350         case NFT_MSG_NEWCHAIN:
10351                 if (nft_trans_chain_update(trans))
10352                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
10353                 else
10354                         nf_tables_chain_destroy(&trans->ctx);
10355                 break;
10356         case NFT_MSG_NEWRULE:
10357                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
10358                 break;
10359         case NFT_MSG_NEWSET:
10360                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
10361                 break;
10362         case NFT_MSG_NEWSETELEM:
10363                 nft_set_elem_destroy(nft_trans_elem_set(trans),
10364                                      nft_trans_elem_priv(trans), true);
10365                 break;
10366         case NFT_MSG_NEWOBJ:
10367                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
10368                 break;
10369         case NFT_MSG_NEWFLOWTABLE:
10370                 if (nft_trans_flowtable_update(trans))
10371                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
10372                 else
10373                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
10374                 break;
10375         }
10376         kfree(trans);
10377 }
10378
10379 static void nft_set_abort_update(struct list_head *set_update_list)
10380 {
10381         struct nft_set *set, *next;
10382
10383         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10384                 list_del_init(&set->pending_update);
10385
10386                 if (!set->ops->abort)
10387                         continue;
10388
10389                 set->ops->abort(set);
10390         }
10391 }
10392
10393 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
10394 {
10395         struct nftables_pernet *nft_net = nft_pernet(net);
10396         struct nft_trans *trans, *next;
10397         LIST_HEAD(set_update_list);
10398         struct nft_trans_elem *te;
10399
10400         if (action == NFNL_ABORT_VALIDATE &&
10401             nf_tables_validate(net) < 0)
10402                 return -EAGAIN;
10403
10404         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
10405                                          list) {
10406                 switch (trans->msg_type) {
10407                 case NFT_MSG_NEWTABLE:
10408                         if (nft_trans_table_update(trans)) {
10409                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10410                                         nft_trans_destroy(trans);
10411                                         break;
10412                                 }
10413                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
10414                                         nf_tables_table_disable(net, trans->ctx.table);
10415                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
10416                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
10417                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
10418                                 }
10419                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10420                                 nft_trans_destroy(trans);
10421                         } else {
10422                                 list_del_rcu(&trans->ctx.table->list);
10423                         }
10424                         break;
10425                 case NFT_MSG_DELTABLE:
10426                 case NFT_MSG_DESTROYTABLE:
10427                         nft_clear(trans->ctx.net, trans->ctx.table);
10428                         nft_trans_destroy(trans);
10429                         break;
10430                 case NFT_MSG_NEWCHAIN:
10431                         if (nft_trans_chain_update(trans)) {
10432                                 nft_netdev_unregister_hooks(net,
10433                                                             &nft_trans_chain_hooks(trans),
10434                                                             true);
10435                                 free_percpu(nft_trans_chain_stats(trans));
10436                                 kfree(nft_trans_chain_name(trans));
10437                                 nft_trans_destroy(trans);
10438                         } else {
10439                                 if (nft_trans_chain_bound(trans)) {
10440                                         nft_trans_destroy(trans);
10441                                         break;
10442                                 }
10443                                 nft_use_dec_restore(&trans->ctx.table->use);
10444                                 nft_chain_del(trans->ctx.chain);
10445                                 nf_tables_unregister_hook(trans->ctx.net,
10446                                                           trans->ctx.table,
10447                                                           trans->ctx.chain);
10448                         }
10449                         break;
10450                 case NFT_MSG_DELCHAIN:
10451                 case NFT_MSG_DESTROYCHAIN:
10452                         if (nft_trans_chain_update(trans)) {
10453                                 list_splice(&nft_trans_chain_hooks(trans),
10454                                             &nft_trans_basechain(trans)->hook_list);
10455                         } else {
10456                                 nft_use_inc_restore(&trans->ctx.table->use);
10457                                 nft_clear(trans->ctx.net, trans->ctx.chain);
10458                         }
10459                         nft_trans_destroy(trans);
10460                         break;
10461                 case NFT_MSG_NEWRULE:
10462                         if (nft_trans_rule_bound(trans)) {
10463                                 nft_trans_destroy(trans);
10464                                 break;
10465                         }
10466                         nft_use_dec_restore(&trans->ctx.chain->use);
10467                         list_del_rcu(&nft_trans_rule(trans)->list);
10468                         nft_rule_expr_deactivate(&trans->ctx,
10469                                                  nft_trans_rule(trans),
10470                                                  NFT_TRANS_ABORT);
10471                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10472                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10473                         break;
10474                 case NFT_MSG_DELRULE:
10475                 case NFT_MSG_DESTROYRULE:
10476                         nft_use_inc_restore(&trans->ctx.chain->use);
10477                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10478                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
10479                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10480                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10481
10482                         nft_trans_destroy(trans);
10483                         break;
10484                 case NFT_MSG_NEWSET:
10485                         if (nft_trans_set_update(trans)) {
10486                                 nft_trans_destroy(trans);
10487                                 break;
10488                         }
10489                         nft_use_dec_restore(&trans->ctx.table->use);
10490                         if (nft_trans_set_bound(trans)) {
10491                                 nft_trans_destroy(trans);
10492                                 break;
10493                         }
10494                         nft_trans_set(trans)->dead = 1;
10495                         list_del_rcu(&nft_trans_set(trans)->list);
10496                         break;
10497                 case NFT_MSG_DELSET:
10498                 case NFT_MSG_DESTROYSET:
10499                         nft_use_inc_restore(&trans->ctx.table->use);
10500                         nft_clear(trans->ctx.net, nft_trans_set(trans));
10501                         if (nft_trans_set(trans)->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
10502                                 nft_map_activate(&trans->ctx, nft_trans_set(trans));
10503
10504                         nft_trans_destroy(trans);
10505                         break;
10506                 case NFT_MSG_NEWSETELEM:
10507                         if (nft_trans_elem_set_bound(trans)) {
10508                                 nft_trans_destroy(trans);
10509                                 break;
10510                         }
10511                         te = (struct nft_trans_elem *)trans->data;
10512                         nft_setelem_remove(net, te->set, te->elem_priv);
10513                         if (!nft_setelem_is_catchall(te->set, te->elem_priv))
10514                                 atomic_dec(&te->set->nelems);
10515
10516                         if (te->set->ops->abort &&
10517                             list_empty(&te->set->pending_update)) {
10518                                 list_add_tail(&te->set->pending_update,
10519                                               &set_update_list);
10520                         }
10521                         break;
10522                 case NFT_MSG_DELSETELEM:
10523                 case NFT_MSG_DESTROYSETELEM:
10524                         te = (struct nft_trans_elem *)trans->data;
10525
10526                         nft_setelem_data_activate(net, te->set, te->elem_priv);
10527                         nft_setelem_activate(net, te->set, te->elem_priv);
10528                         if (!nft_setelem_is_catchall(te->set, te->elem_priv))
10529                                 te->set->ndeact--;
10530
10531                         if (te->set->ops->abort &&
10532                             list_empty(&te->set->pending_update)) {
10533                                 list_add_tail(&te->set->pending_update,
10534                                               &set_update_list);
10535                         }
10536                         nft_trans_destroy(trans);
10537                         break;
10538                 case NFT_MSG_NEWOBJ:
10539                         if (nft_trans_obj_update(trans)) {
10540                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
10541                                 nft_trans_destroy(trans);
10542                         } else {
10543                                 nft_use_dec_restore(&trans->ctx.table->use);
10544                                 nft_obj_del(nft_trans_obj(trans));
10545                         }
10546                         break;
10547                 case NFT_MSG_DELOBJ:
10548                 case NFT_MSG_DESTROYOBJ:
10549                         nft_use_inc_restore(&trans->ctx.table->use);
10550                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
10551                         nft_trans_destroy(trans);
10552                         break;
10553                 case NFT_MSG_NEWFLOWTABLE:
10554                         if (nft_trans_flowtable_update(trans)) {
10555                                 nft_unregister_flowtable_net_hooks(net,
10556                                                 &nft_trans_flowtable_hooks(trans));
10557                         } else {
10558                                 nft_use_dec_restore(&trans->ctx.table->use);
10559                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10560                                 nft_unregister_flowtable_net_hooks(net,
10561                                                 &nft_trans_flowtable(trans)->hook_list);
10562                         }
10563                         break;
10564                 case NFT_MSG_DELFLOWTABLE:
10565                 case NFT_MSG_DESTROYFLOWTABLE:
10566                         if (nft_trans_flowtable_update(trans)) {
10567                                 list_splice(&nft_trans_flowtable_hooks(trans),
10568                                             &nft_trans_flowtable(trans)->hook_list);
10569                         } else {
10570                                 nft_use_inc_restore(&trans->ctx.table->use);
10571                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
10572                         }
10573                         nft_trans_destroy(trans);
10574                         break;
10575                 }
10576         }
10577
10578         nft_set_abort_update(&set_update_list);
10579
10580         synchronize_rcu();
10581
10582         list_for_each_entry_safe_reverse(trans, next,
10583                                          &nft_net->commit_list, list) {
10584                 nft_trans_list_del(trans);
10585                 nf_tables_abort_release(trans);
10586         }
10587
10588         if (action == NFNL_ABORT_AUTOLOAD)
10589                 nf_tables_module_autoload(net);
10590         else
10591                 nf_tables_module_autoload_cleanup(net);
10592
10593         return 0;
10594 }
10595
10596 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
10597                            enum nfnl_abort_action action)
10598 {
10599         struct nftables_pernet *nft_net = nft_pernet(net);
10600         unsigned int gc_seq;
10601         int ret;
10602
10603         gc_seq = nft_gc_seq_begin(nft_net);
10604         ret = __nf_tables_abort(net, action);
10605         nft_gc_seq_end(nft_net, gc_seq);
10606         mutex_unlock(&nft_net->commit_mutex);
10607
10608         return ret;
10609 }
10610
10611 static bool nf_tables_valid_genid(struct net *net, u32 genid)
10612 {
10613         struct nftables_pernet *nft_net = nft_pernet(net);
10614         bool genid_ok;
10615
10616         mutex_lock(&nft_net->commit_mutex);
10617
10618         genid_ok = genid == 0 || nft_net->base_seq == genid;
10619         if (!genid_ok)
10620                 mutex_unlock(&nft_net->commit_mutex);
10621
10622         /* else, commit mutex has to be released by commit or abort function */
10623         return genid_ok;
10624 }
10625
10626 static const struct nfnetlink_subsystem nf_tables_subsys = {
10627         .name           = "nf_tables",
10628         .subsys_id      = NFNL_SUBSYS_NFTABLES,
10629         .cb_count       = NFT_MSG_MAX,
10630         .cb             = nf_tables_cb,
10631         .commit         = nf_tables_commit,
10632         .abort          = nf_tables_abort,
10633         .valid_genid    = nf_tables_valid_genid,
10634         .owner          = THIS_MODULE,
10635 };
10636
10637 int nft_chain_validate_dependency(const struct nft_chain *chain,
10638                                   enum nft_chain_types type)
10639 {
10640         const struct nft_base_chain *basechain;
10641
10642         if (nft_is_base_chain(chain)) {
10643                 basechain = nft_base_chain(chain);
10644                 if (basechain->type->type != type)
10645                         return -EOPNOTSUPP;
10646         }
10647         return 0;
10648 }
10649 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
10650
10651 int nft_chain_validate_hooks(const struct nft_chain *chain,
10652                              unsigned int hook_flags)
10653 {
10654         struct nft_base_chain *basechain;
10655
10656         if (nft_is_base_chain(chain)) {
10657                 basechain = nft_base_chain(chain);
10658
10659                 if ((1 << basechain->ops.hooknum) & hook_flags)
10660                         return 0;
10661
10662                 return -EOPNOTSUPP;
10663         }
10664
10665         return 0;
10666 }
10667 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
10668
10669 /*
10670  * Loop detection - walk through the ruleset beginning at the destination chain
10671  * of a new jump until either the source chain is reached (loop) or all
10672  * reachable chains have been traversed.
10673  *
10674  * The loop check is performed whenever a new jump verdict is added to an
10675  * expression or verdict map or a verdict map is bound to a new chain.
10676  */
10677
10678 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10679                                  const struct nft_chain *chain);
10680
10681 static int nft_check_loops(const struct nft_ctx *ctx,
10682                            const struct nft_set_ext *ext)
10683 {
10684         const struct nft_data *data;
10685         int ret;
10686
10687         data = nft_set_ext_data(ext);
10688         switch (data->verdict.code) {
10689         case NFT_JUMP:
10690         case NFT_GOTO:
10691                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
10692                 break;
10693         default:
10694                 ret = 0;
10695                 break;
10696         }
10697
10698         return ret;
10699 }
10700
10701 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
10702                                         struct nft_set *set,
10703                                         const struct nft_set_iter *iter,
10704                                         struct nft_elem_priv *elem_priv)
10705 {
10706         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
10707
10708         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
10709             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
10710                 return 0;
10711
10712         return nft_check_loops(ctx, ext);
10713 }
10714
10715 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
10716                                   struct nft_set *set)
10717 {
10718         u8 genmask = nft_genmask_next(ctx->net);
10719         struct nft_set_elem_catchall *catchall;
10720         struct nft_set_ext *ext;
10721         int ret = 0;
10722
10723         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10724                 ext = nft_set_elem_ext(set, catchall->elem);
10725                 if (!nft_set_elem_active(ext, genmask))
10726                         continue;
10727
10728                 ret = nft_check_loops(ctx, ext);
10729                 if (ret < 0)
10730                         return ret;
10731         }
10732
10733         return ret;
10734 }
10735
10736 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10737                                  const struct nft_chain *chain)
10738 {
10739         const struct nft_rule *rule;
10740         const struct nft_expr *expr, *last;
10741         struct nft_set *set;
10742         struct nft_set_binding *binding;
10743         struct nft_set_iter iter;
10744
10745         if (ctx->chain == chain)
10746                 return -ELOOP;
10747
10748         if (fatal_signal_pending(current))
10749                 return -EINTR;
10750
10751         list_for_each_entry(rule, &chain->rules, list) {
10752                 nft_rule_for_each_expr(expr, last, rule) {
10753                         struct nft_immediate_expr *priv;
10754                         const struct nft_data *data;
10755                         int err;
10756
10757                         if (strcmp(expr->ops->type->name, "immediate"))
10758                                 continue;
10759
10760                         priv = nft_expr_priv(expr);
10761                         if (priv->dreg != NFT_REG_VERDICT)
10762                                 continue;
10763
10764                         data = &priv->data;
10765                         switch (data->verdict.code) {
10766                         case NFT_JUMP:
10767                         case NFT_GOTO:
10768                                 err = nf_tables_check_loops(ctx,
10769                                                         data->verdict.chain);
10770                                 if (err < 0)
10771                                         return err;
10772                                 break;
10773                         default:
10774                                 break;
10775                         }
10776                 }
10777         }
10778
10779         list_for_each_entry(set, &ctx->table->sets, list) {
10780                 if (!nft_is_active_next(ctx->net, set))
10781                         continue;
10782                 if (!(set->flags & NFT_SET_MAP) ||
10783                     set->dtype != NFT_DATA_VERDICT)
10784                         continue;
10785
10786                 list_for_each_entry(binding, &set->bindings, list) {
10787                         if (!(binding->flags & NFT_SET_MAP) ||
10788                             binding->chain != chain)
10789                                 continue;
10790
10791                         iter.genmask    = nft_genmask_next(ctx->net);
10792                         iter.skip       = 0;
10793                         iter.count      = 0;
10794                         iter.err        = 0;
10795                         iter.fn         = nf_tables_loop_check_setelem;
10796
10797                         set->ops->walk(ctx, set, &iter);
10798                         if (!iter.err)
10799                                 iter.err = nft_set_catchall_loops(ctx, set);
10800
10801                         if (iter.err < 0)
10802                                 return iter.err;
10803                 }
10804         }
10805
10806         return 0;
10807 }
10808
10809 /**
10810  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
10811  *
10812  *      @attr: netlink attribute to fetch value from
10813  *      @max: maximum value to be stored in dest
10814  *      @dest: pointer to the variable
10815  *
10816  *      Parse, check and store a given u32 netlink attribute into variable.
10817  *      This function returns -ERANGE if the value goes over maximum value.
10818  *      Otherwise a 0 is returned and the attribute value is stored in the
10819  *      destination variable.
10820  */
10821 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
10822 {
10823         u32 val;
10824
10825         val = ntohl(nla_get_be32(attr));
10826         if (val > max)
10827                 return -ERANGE;
10828
10829         *dest = val;
10830         return 0;
10831 }
10832 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
10833
10834 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
10835 {
10836         unsigned int reg;
10837
10838         reg = ntohl(nla_get_be32(attr));
10839         switch (reg) {
10840         case NFT_REG_VERDICT...NFT_REG_4:
10841                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
10842                 break;
10843         case NFT_REG32_00...NFT_REG32_15:
10844                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
10845                 break;
10846         default:
10847                 return -ERANGE;
10848         }
10849
10850         return 0;
10851 }
10852
10853 /**
10854  *      nft_dump_register - dump a register value to a netlink attribute
10855  *
10856  *      @skb: socket buffer
10857  *      @attr: attribute number
10858  *      @reg: register number
10859  *
10860  *      Construct a netlink attribute containing the register number. For
10861  *      compatibility reasons, register numbers being a multiple of 4 are
10862  *      translated to the corresponding 128 bit register numbers.
10863  */
10864 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
10865 {
10866         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
10867                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
10868         else
10869                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
10870
10871         return nla_put_be32(skb, attr, htonl(reg));
10872 }
10873 EXPORT_SYMBOL_GPL(nft_dump_register);
10874
10875 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
10876 {
10877         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10878                 return -EINVAL;
10879         if (len == 0)
10880                 return -EINVAL;
10881         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
10882                 return -ERANGE;
10883
10884         return 0;
10885 }
10886
10887 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
10888 {
10889         u32 reg;
10890         int err;
10891
10892         err = nft_parse_register(attr, &reg);
10893         if (err < 0)
10894                 return err;
10895
10896         err = nft_validate_register_load(reg, len);
10897         if (err < 0)
10898                 return err;
10899
10900         *sreg = reg;
10901         return 0;
10902 }
10903 EXPORT_SYMBOL_GPL(nft_parse_register_load);
10904
10905 static int nft_validate_register_store(const struct nft_ctx *ctx,
10906                                        enum nft_registers reg,
10907                                        const struct nft_data *data,
10908                                        enum nft_data_types type,
10909                                        unsigned int len)
10910 {
10911         int err;
10912
10913         switch (reg) {
10914         case NFT_REG_VERDICT:
10915                 if (type != NFT_DATA_VERDICT)
10916                         return -EINVAL;
10917
10918                 if (data != NULL &&
10919                     (data->verdict.code == NFT_GOTO ||
10920                      data->verdict.code == NFT_JUMP)) {
10921                         err = nf_tables_check_loops(ctx, data->verdict.chain);
10922                         if (err < 0)
10923                                 return err;
10924                 }
10925
10926                 return 0;
10927         default:
10928                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
10929                         return -EINVAL;
10930                 if (len == 0)
10931                         return -EINVAL;
10932                 if (reg * NFT_REG32_SIZE + len >
10933                     sizeof_field(struct nft_regs, data))
10934                         return -ERANGE;
10935
10936                 if (data != NULL && type != NFT_DATA_VALUE)
10937                         return -EINVAL;
10938                 return 0;
10939         }
10940 }
10941
10942 int nft_parse_register_store(const struct nft_ctx *ctx,
10943                              const struct nlattr *attr, u8 *dreg,
10944                              const struct nft_data *data,
10945                              enum nft_data_types type, unsigned int len)
10946 {
10947         int err;
10948         u32 reg;
10949
10950         err = nft_parse_register(attr, &reg);
10951         if (err < 0)
10952                 return err;
10953
10954         err = nft_validate_register_store(ctx, reg, data, type, len);
10955         if (err < 0)
10956                 return err;
10957
10958         *dreg = reg;
10959         return 0;
10960 }
10961 EXPORT_SYMBOL_GPL(nft_parse_register_store);
10962
10963 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
10964         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
10965         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
10966                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
10967         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
10968 };
10969
10970 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
10971                             struct nft_data_desc *desc, const struct nlattr *nla)
10972 {
10973         u8 genmask = nft_genmask_next(ctx->net);
10974         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
10975         struct nft_chain *chain;
10976         int err;
10977
10978         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
10979                                           nft_verdict_policy, NULL);
10980         if (err < 0)
10981                 return err;
10982
10983         if (!tb[NFTA_VERDICT_CODE])
10984                 return -EINVAL;
10985
10986         /* zero padding hole for memcmp */
10987         memset(data, 0, sizeof(*data));
10988         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
10989
10990         switch (data->verdict.code) {
10991         default:
10992                 switch (data->verdict.code & NF_VERDICT_MASK) {
10993                 case NF_ACCEPT:
10994                 case NF_DROP:
10995                 case NF_QUEUE:
10996                         break;
10997                 default:
10998                         return -EINVAL;
10999                 }
11000                 fallthrough;
11001         case NFT_CONTINUE:
11002         case NFT_BREAK:
11003         case NFT_RETURN:
11004                 break;
11005         case NFT_JUMP:
11006         case NFT_GOTO:
11007                 if (tb[NFTA_VERDICT_CHAIN]) {
11008                         chain = nft_chain_lookup(ctx->net, ctx->table,
11009                                                  tb[NFTA_VERDICT_CHAIN],
11010                                                  genmask);
11011                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
11012                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
11013                                                       tb[NFTA_VERDICT_CHAIN_ID],
11014                                                       genmask);
11015                         if (IS_ERR(chain))
11016                                 return PTR_ERR(chain);
11017                 } else {
11018                         return -EINVAL;
11019                 }
11020
11021                 if (IS_ERR(chain))
11022                         return PTR_ERR(chain);
11023                 if (nft_is_base_chain(chain))
11024                         return -EOPNOTSUPP;
11025                 if (nft_chain_is_bound(chain))
11026                         return -EINVAL;
11027                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
11028                     chain->flags & NFT_CHAIN_BINDING)
11029                         return -EINVAL;
11030                 if (!nft_use_inc(&chain->use))
11031                         return -EMFILE;
11032
11033                 data->verdict.chain = chain;
11034                 break;
11035         }
11036
11037         desc->len = sizeof(data->verdict);
11038
11039         return 0;
11040 }
11041
11042 static void nft_verdict_uninit(const struct nft_data *data)
11043 {
11044         struct nft_chain *chain;
11045
11046         switch (data->verdict.code) {
11047         case NFT_JUMP:
11048         case NFT_GOTO:
11049                 chain = data->verdict.chain;
11050                 nft_use_dec(&chain->use);
11051                 break;
11052         }
11053 }
11054
11055 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
11056 {
11057         struct nlattr *nest;
11058
11059         nest = nla_nest_start_noflag(skb, type);
11060         if (!nest)
11061                 goto nla_put_failure;
11062
11063         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
11064                 goto nla_put_failure;
11065
11066         switch (v->code) {
11067         case NFT_JUMP:
11068         case NFT_GOTO:
11069                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
11070                                    v->chain->name))
11071                         goto nla_put_failure;
11072         }
11073         nla_nest_end(skb, nest);
11074         return 0;
11075
11076 nla_put_failure:
11077         return -1;
11078 }
11079
11080 static int nft_value_init(const struct nft_ctx *ctx,
11081                           struct nft_data *data, struct nft_data_desc *desc,
11082                           const struct nlattr *nla)
11083 {
11084         unsigned int len;
11085
11086         len = nla_len(nla);
11087         if (len == 0)
11088                 return -EINVAL;
11089         if (len > desc->size)
11090                 return -EOVERFLOW;
11091         if (desc->len) {
11092                 if (len != desc->len)
11093                         return -EINVAL;
11094         } else {
11095                 desc->len = len;
11096         }
11097
11098         nla_memcpy(data->data, nla, len);
11099
11100         return 0;
11101 }
11102
11103 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
11104                           unsigned int len)
11105 {
11106         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
11107 }
11108
11109 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
11110         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
11111         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
11112 };
11113
11114 /**
11115  *      nft_data_init - parse nf_tables data netlink attributes
11116  *
11117  *      @ctx: context of the expression using the data
11118  *      @data: destination struct nft_data
11119  *      @desc: data description
11120  *      @nla: netlink attribute containing data
11121  *
11122  *      Parse the netlink data attributes and initialize a struct nft_data.
11123  *      The type and length of data are returned in the data description.
11124  *
11125  *      The caller can indicate that it only wants to accept data of type
11126  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
11127  */
11128 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
11129                   struct nft_data_desc *desc, const struct nlattr *nla)
11130 {
11131         struct nlattr *tb[NFTA_DATA_MAX + 1];
11132         int err;
11133
11134         if (WARN_ON_ONCE(!desc->size))
11135                 return -EINVAL;
11136
11137         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
11138                                           nft_data_policy, NULL);
11139         if (err < 0)
11140                 return err;
11141
11142         if (tb[NFTA_DATA_VALUE]) {
11143                 if (desc->type != NFT_DATA_VALUE)
11144                         return -EINVAL;
11145
11146                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
11147         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
11148                 if (desc->type != NFT_DATA_VERDICT)
11149                         return -EINVAL;
11150
11151                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
11152         } else {
11153                 err = -EINVAL;
11154         }
11155
11156         return err;
11157 }
11158 EXPORT_SYMBOL_GPL(nft_data_init);
11159
11160 /**
11161  *      nft_data_release - release a nft_data item
11162  *
11163  *      @data: struct nft_data to release
11164  *      @type: type of data
11165  *
11166  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
11167  *      all others need to be released by calling this function.
11168  */
11169 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
11170 {
11171         if (type < NFT_DATA_VERDICT)
11172                 return;
11173         switch (type) {
11174         case NFT_DATA_VERDICT:
11175                 return nft_verdict_uninit(data);
11176         default:
11177                 WARN_ON(1);
11178         }
11179 }
11180 EXPORT_SYMBOL_GPL(nft_data_release);
11181
11182 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
11183                   enum nft_data_types type, unsigned int len)
11184 {
11185         struct nlattr *nest;
11186         int err;
11187
11188         nest = nla_nest_start_noflag(skb, attr);
11189         if (nest == NULL)
11190                 return -1;
11191
11192         switch (type) {
11193         case NFT_DATA_VALUE:
11194                 err = nft_value_dump(skb, data, len);
11195                 break;
11196         case NFT_DATA_VERDICT:
11197                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
11198                 break;
11199         default:
11200                 err = -EINVAL;
11201                 WARN_ON(1);
11202         }
11203
11204         nla_nest_end(skb, nest);
11205         return err;
11206 }
11207 EXPORT_SYMBOL_GPL(nft_data_dump);
11208
11209 int __nft_release_basechain(struct nft_ctx *ctx)
11210 {
11211         struct nft_rule *rule, *nr;
11212
11213         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
11214                 return 0;
11215
11216         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
11217         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
11218                 list_del(&rule->list);
11219                 nft_use_dec(&ctx->chain->use);
11220                 nf_tables_rule_release(ctx, rule);
11221         }
11222         nft_chain_del(ctx->chain);
11223         nft_use_dec(&ctx->table->use);
11224         nf_tables_chain_destroy(ctx);
11225
11226         return 0;
11227 }
11228 EXPORT_SYMBOL_GPL(__nft_release_basechain);
11229
11230 static void __nft_release_hook(struct net *net, struct nft_table *table)
11231 {
11232         struct nft_flowtable *flowtable;
11233         struct nft_chain *chain;
11234
11235         list_for_each_entry(chain, &table->chains, list)
11236                 __nf_tables_unregister_hook(net, table, chain, true);
11237         list_for_each_entry(flowtable, &table->flowtables, list)
11238                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
11239                                                      true);
11240 }
11241
11242 static void __nft_release_hooks(struct net *net)
11243 {
11244         struct nftables_pernet *nft_net = nft_pernet(net);
11245         struct nft_table *table;
11246
11247         list_for_each_entry(table, &nft_net->tables, list) {
11248                 if (nft_table_has_owner(table))
11249                         continue;
11250
11251                 __nft_release_hook(net, table);
11252         }
11253 }
11254
11255 static void __nft_release_table(struct net *net, struct nft_table *table)
11256 {
11257         struct nft_flowtable *flowtable, *nf;
11258         struct nft_chain *chain, *nc;
11259         struct nft_object *obj, *ne;
11260         struct nft_rule *rule, *nr;
11261         struct nft_set *set, *ns;
11262         struct nft_ctx ctx = {
11263                 .net    = net,
11264                 .family = NFPROTO_NETDEV,
11265         };
11266
11267         ctx.family = table->family;
11268         ctx.table = table;
11269         list_for_each_entry(chain, &table->chains, list) {
11270                 if (nft_chain_binding(chain))
11271                         continue;
11272
11273                 ctx.chain = chain;
11274                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
11275                         list_del(&rule->list);
11276                         nft_use_dec(&chain->use);
11277                         nf_tables_rule_release(&ctx, rule);
11278                 }
11279         }
11280         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
11281                 list_del(&flowtable->list);
11282                 nft_use_dec(&table->use);
11283                 nf_tables_flowtable_destroy(flowtable);
11284         }
11285         list_for_each_entry_safe(set, ns, &table->sets, list) {
11286                 list_del(&set->list);
11287                 nft_use_dec(&table->use);
11288                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11289                         nft_map_deactivate(&ctx, set);
11290
11291                 nft_set_destroy(&ctx, set);
11292         }
11293         list_for_each_entry_safe(obj, ne, &table->objects, list) {
11294                 nft_obj_del(obj);
11295                 nft_use_dec(&table->use);
11296                 nft_obj_destroy(&ctx, obj);
11297         }
11298         list_for_each_entry_safe(chain, nc, &table->chains, list) {
11299                 ctx.chain = chain;
11300                 nft_chain_del(chain);
11301                 nft_use_dec(&table->use);
11302                 nf_tables_chain_destroy(&ctx);
11303         }
11304         nf_tables_table_destroy(&ctx);
11305 }
11306
11307 static void __nft_release_tables(struct net *net)
11308 {
11309         struct nftables_pernet *nft_net = nft_pernet(net);
11310         struct nft_table *table, *nt;
11311
11312         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
11313                 if (nft_table_has_owner(table))
11314                         continue;
11315
11316                 list_del(&table->list);
11317
11318                 __nft_release_table(net, table);
11319         }
11320 }
11321
11322 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
11323                             void *ptr)
11324 {
11325         struct nft_table *table, *to_delete[8];
11326         struct nftables_pernet *nft_net;
11327         struct netlink_notify *n = ptr;
11328         struct net *net = n->net;
11329         unsigned int deleted;
11330         bool restart = false;
11331         unsigned int gc_seq;
11332
11333         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
11334                 return NOTIFY_DONE;
11335
11336         nft_net = nft_pernet(net);
11337         deleted = 0;
11338         mutex_lock(&nft_net->commit_mutex);
11339
11340         gc_seq = nft_gc_seq_begin(nft_net);
11341
11342         if (!list_empty(&nf_tables_destroy_list))
11343                 nf_tables_trans_destroy_flush_work();
11344 again:
11345         list_for_each_entry(table, &nft_net->tables, list) {
11346                 if (nft_table_has_owner(table) &&
11347                     n->portid == table->nlpid) {
11348                         __nft_release_hook(net, table);
11349                         list_del_rcu(&table->list);
11350                         to_delete[deleted++] = table;
11351                         if (deleted >= ARRAY_SIZE(to_delete))
11352                                 break;
11353                 }
11354         }
11355         if (deleted) {
11356                 restart = deleted >= ARRAY_SIZE(to_delete);
11357                 synchronize_rcu();
11358                 while (deleted)
11359                         __nft_release_table(net, to_delete[--deleted]);
11360
11361                 if (restart)
11362                         goto again;
11363         }
11364         nft_gc_seq_end(nft_net, gc_seq);
11365
11366         mutex_unlock(&nft_net->commit_mutex);
11367
11368         return NOTIFY_DONE;
11369 }
11370
11371 static struct notifier_block nft_nl_notifier = {
11372         .notifier_call  = nft_rcv_nl_event,
11373 };
11374
11375 static int __net_init nf_tables_init_net(struct net *net)
11376 {
11377         struct nftables_pernet *nft_net = nft_pernet(net);
11378
11379         INIT_LIST_HEAD(&nft_net->tables);
11380         INIT_LIST_HEAD(&nft_net->commit_list);
11381         INIT_LIST_HEAD(&nft_net->binding_list);
11382         INIT_LIST_HEAD(&nft_net->module_list);
11383         INIT_LIST_HEAD(&nft_net->notify_list);
11384         mutex_init(&nft_net->commit_mutex);
11385         nft_net->base_seq = 1;
11386         nft_net->gc_seq = 0;
11387         nft_net->validate_state = NFT_VALIDATE_SKIP;
11388
11389         return 0;
11390 }
11391
11392 static void __net_exit nf_tables_pre_exit_net(struct net *net)
11393 {
11394         struct nftables_pernet *nft_net = nft_pernet(net);
11395
11396         mutex_lock(&nft_net->commit_mutex);
11397         __nft_release_hooks(net);
11398         mutex_unlock(&nft_net->commit_mutex);
11399 }
11400
11401 static void __net_exit nf_tables_exit_net(struct net *net)
11402 {
11403         struct nftables_pernet *nft_net = nft_pernet(net);
11404         unsigned int gc_seq;
11405
11406         mutex_lock(&nft_net->commit_mutex);
11407
11408         gc_seq = nft_gc_seq_begin(nft_net);
11409
11410         if (!list_empty(&nft_net->commit_list) ||
11411             !list_empty(&nft_net->module_list))
11412                 __nf_tables_abort(net, NFNL_ABORT_NONE);
11413
11414         __nft_release_tables(net);
11415
11416         nft_gc_seq_end(nft_net, gc_seq);
11417
11418         mutex_unlock(&nft_net->commit_mutex);
11419         WARN_ON_ONCE(!list_empty(&nft_net->tables));
11420         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
11421         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
11422 }
11423
11424 static void nf_tables_exit_batch(struct list_head *net_exit_list)
11425 {
11426         flush_work(&trans_gc_work);
11427 }
11428
11429 static struct pernet_operations nf_tables_net_ops = {
11430         .init           = nf_tables_init_net,
11431         .pre_exit       = nf_tables_pre_exit_net,
11432         .exit           = nf_tables_exit_net,
11433         .exit_batch     = nf_tables_exit_batch,
11434         .id             = &nf_tables_net_id,
11435         .size           = sizeof(struct nftables_pernet),
11436 };
11437
11438 static int __init nf_tables_module_init(void)
11439 {
11440         int err;
11441
11442         err = register_pernet_subsys(&nf_tables_net_ops);
11443         if (err < 0)
11444                 return err;
11445
11446         err = nft_chain_filter_init();
11447         if (err < 0)
11448                 goto err_chain_filter;
11449
11450         err = nf_tables_core_module_init();
11451         if (err < 0)
11452                 goto err_core_module;
11453
11454         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
11455         if (err < 0)
11456                 goto err_netdev_notifier;
11457
11458         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
11459         if (err < 0)
11460                 goto err_rht_objname;
11461
11462         err = nft_offload_init();
11463         if (err < 0)
11464                 goto err_offload;
11465
11466         err = netlink_register_notifier(&nft_nl_notifier);
11467         if (err < 0)
11468                 goto err_netlink_notifier;
11469
11470         /* must be last */
11471         err = nfnetlink_subsys_register(&nf_tables_subsys);
11472         if (err < 0)
11473                 goto err_nfnl_subsys;
11474
11475         nft_chain_route_init();
11476
11477         return err;
11478
11479 err_nfnl_subsys:
11480         netlink_unregister_notifier(&nft_nl_notifier);
11481 err_netlink_notifier:
11482         nft_offload_exit();
11483 err_offload:
11484         rhltable_destroy(&nft_objname_ht);
11485 err_rht_objname:
11486         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11487 err_netdev_notifier:
11488         nf_tables_core_module_exit();
11489 err_core_module:
11490         nft_chain_filter_fini();
11491 err_chain_filter:
11492         unregister_pernet_subsys(&nf_tables_net_ops);
11493         return err;
11494 }
11495
11496 static void __exit nf_tables_module_exit(void)
11497 {
11498         nfnetlink_subsys_unregister(&nf_tables_subsys);
11499         netlink_unregister_notifier(&nft_nl_notifier);
11500         nft_offload_exit();
11501         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11502         nft_chain_filter_fini();
11503         nft_chain_route_fini();
11504         unregister_pernet_subsys(&nf_tables_net_ops);
11505         cancel_work_sync(&trans_gc_work);
11506         cancel_work_sync(&trans_destroy_work);
11507         rcu_barrier();
11508         rhltable_destroy(&nft_objname_ht);
11509         nf_tables_core_module_exit();
11510 }
11511
11512 module_init(nf_tables_module_init);
11513 module_exit(nf_tables_module_exit);
11514
11515 MODULE_LICENSE("GPL");
11516 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
11517 MODULE_DESCRIPTION("Framework for packet filtering and classification");
11518 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);