net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27 #define NFT_SET_MAX_ANONLEN 16
  28
  29 unsigned int nf_tables_net_id __read_mostly;
  30
  31 static LIST_HEAD(nf_tables_expressions);
  32 static LIST_HEAD(nf_tables_objects);
  33 static LIST_HEAD(nf_tables_flowtables);
  34 static LIST_HEAD(nf_tables_destroy_list);
  35 static LIST_HEAD(nf_tables_gc_list);
  36 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  37 static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
  38
  39 enum {
  40         NFT_VALIDATE_SKIP       = 0,
  41         NFT_VALIDATE_NEED,
  42         NFT_VALIDATE_DO,
  43 };
  44
  45 static struct rhltable nft_objname_ht;
  46
  47 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  48 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  49 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  50
  51 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  52 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  53 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  54
  55 static const struct rhashtable_params nft_chain_ht_params = {
  56         .head_offset            = offsetof(struct nft_chain, rhlhead),
  57         .key_offset             = offsetof(struct nft_chain, name),
  58         .hashfn                 = nft_chain_hash,
  59         .obj_hashfn             = nft_chain_hash_obj,
  60         .obj_cmpfn              = nft_chain_hash_cmp,
  61         .automatic_shrinking    = true,
  62 };
  63
  64 static const struct rhashtable_params nft_objname_ht_params = {
  65         .head_offset            = offsetof(struct nft_object, rhlhead),
  66         .key_offset             = offsetof(struct nft_object, key),
  67         .hashfn                 = nft_objname_hash,
  68         .obj_hashfn             = nft_objname_hash_obj,
  69         .obj_cmpfn              = nft_objname_hash_cmp,
  70         .automatic_shrinking    = true,
  71 };
  72
  73 struct nft_audit_data {
  74         struct nft_table *table;
  75         int entries;
  76         int op;
  77         struct list_head list;
  78 };
  79
  80 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  81         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  82         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  83         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  84         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  85         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  86         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  87         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  88         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  89         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  90         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  91         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  92         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  93         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  94         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  95         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  96         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  97         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  98         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  99         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
 100         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
 101         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
 102         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 103         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 104         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 105         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 106         [NFT_MSG_GETSETELEM_RESET] = AUDIT_NFT_OP_SETELEM_RESET,
 107 };
 108
 109 static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
 110 {
 111         switch (table->validate_state) {
 112         case NFT_VALIDATE_SKIP:
 113                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 114                 break;
 115         case NFT_VALIDATE_NEED:
 116                 break;
 117         case NFT_VALIDATE_DO:
 118                 if (new_validate_state == NFT_VALIDATE_NEED)
 119                         return;
 120         }
 121
 122         table->validate_state = new_validate_state;
 123 }
 124 static void nf_tables_trans_destroy_work(struct work_struct *w);
 125 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 126
 127 static void nft_trans_gc_work(struct work_struct *work);
 128 static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
 129
 130 static void nft_ctx_init(struct nft_ctx *ctx,
 131                          struct net *net,
 132                          const struct sk_buff *skb,
 133                          const struct nlmsghdr *nlh,
 134                          u8 family,
 135                          struct nft_table *table,
 136                          struct nft_chain *chain,
 137                          const struct nlattr * const *nla)
 138 {
 139         ctx->net        = net;
 140         ctx->family     = family;
 141         ctx->level      = 0;
 142         ctx->table      = table;
 143         ctx->chain      = chain;
 144         ctx->nla        = nla;
 145         ctx->portid     = NETLINK_CB(skb).portid;
 146         ctx->report     = nlmsg_report(nlh);
 147         ctx->flags      = nlh->nlmsg_flags;
 148         ctx->seq        = nlh->nlmsg_seq;
 149 }
 150
 151 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 152                                              int msg_type, u32 size, gfp_t gfp)
 153 {
 154         struct nft_trans *trans;
 155
 156         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 157         if (trans == NULL)
 158                 return NULL;
 159
 160         INIT_LIST_HEAD(&trans->list);
 161         INIT_LIST_HEAD(&trans->binding_list);
 162         trans->msg_type = msg_type;
 163         trans->ctx      = *ctx;
 164
 165         return trans;
 166 }
 167
 168 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 169                                          int msg_type, u32 size)
 170 {
 171         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 172 }
 173
 174 static void nft_trans_list_del(struct nft_trans *trans)
 175 {
 176         list_del(&trans->list);
 177         list_del(&trans->binding_list);
 178 }
 179
 180 static void nft_trans_destroy(struct nft_trans *trans)
 181 {
 182         nft_trans_list_del(trans);
 183         kfree(trans);
 184 }
 185
 186 static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
 187                                  bool bind)
 188 {
 189         struct nftables_pernet *nft_net;
 190         struct net *net = ctx->net;
 191         struct nft_trans *trans;
 192
 193         if (!nft_set_is_anonymous(set))
 194                 return;
 195
 196         nft_net = nft_pernet(net);
 197         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 198                 switch (trans->msg_type) {
 199                 case NFT_MSG_NEWSET:
 200                         if (nft_trans_set(trans) == set)
 201                                 nft_trans_set_bound(trans) = bind;
 202                         break;
 203                 case NFT_MSG_NEWSETELEM:
 204                         if (nft_trans_elem_set(trans) == set)
 205                                 nft_trans_elem_set_bound(trans) = bind;
 206                         break;
 207                 }
 208         }
 209 }
 210
 211 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 212 {
 213         return __nft_set_trans_bind(ctx, set, true);
 214 }
 215
 216 static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
 217 {
 218         return __nft_set_trans_bind(ctx, set, false);
 219 }
 220
 221 static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
 222                                    struct nft_chain *chain, bool bind)
 223 {
 224         struct nftables_pernet *nft_net;
 225         struct net *net = ctx->net;
 226         struct nft_trans *trans;
 227
 228         if (!nft_chain_binding(chain))
 229                 return;
 230
 231         nft_net = nft_pernet(net);
 232         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 233                 switch (trans->msg_type) {
 234                 case NFT_MSG_NEWCHAIN:
 235                         if (nft_trans_chain(trans) == chain)
 236                                 nft_trans_chain_bound(trans) = bind;
 237                         break;
 238                 case NFT_MSG_NEWRULE:
 239                         if (trans->ctx.chain == chain)
 240                                 nft_trans_rule_bound(trans) = bind;
 241                         break;
 242                 }
 243         }
 244 }
 245
 246 static void nft_chain_trans_bind(const struct nft_ctx *ctx,
 247                                  struct nft_chain *chain)
 248 {
 249         __nft_chain_trans_bind(ctx, chain, true);
 250 }
 251
 252 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 253 {
 254         if (!nft_chain_binding(chain))
 255                 return 0;
 256
 257         if (nft_chain_binding(ctx->chain))
 258                 return -EOPNOTSUPP;
 259
 260         if (chain->bound)
 261                 return -EBUSY;
 262
 263         if (!nft_use_inc(&chain->use))
 264                 return -EMFILE;
 265
 266         chain->bound = true;
 267         nft_chain_trans_bind(ctx, chain);
 268
 269         return 0;
 270 }
 271
 272 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
 273 {
 274         __nft_chain_trans_bind(ctx, chain, false);
 275 }
 276
 277 static int nft_netdev_register_hooks(struct net *net,
 278                                      struct list_head *hook_list)
 279 {
 280         struct nft_hook *hook;
 281         int err, j;
 282
 283         j = 0;
 284         list_for_each_entry(hook, hook_list, list) {
 285                 err = nf_register_net_hook(net, &hook->ops);
 286                 if (err < 0)
 287                         goto err_register;
 288
 289                 j++;
 290         }
 291         return 0;
 292
 293 err_register:
 294         list_for_each_entry(hook, hook_list, list) {
 295                 if (j-- <= 0)
 296                         break;
 297
 298                 nf_unregister_net_hook(net, &hook->ops);
 299         }
 300         return err;
 301 }
 302
 303 static void nft_netdev_unregister_hooks(struct net *net,
 304                                         struct list_head *hook_list,
 305                                         bool release_netdev)
 306 {
 307         struct nft_hook *hook, *next;
 308
 309         list_for_each_entry_safe(hook, next, hook_list, list) {
 310                 nf_unregister_net_hook(net, &hook->ops);
 311                 if (release_netdev) {
 312                         list_del(&hook->list);
 313                         kfree_rcu(hook, rcu);
 314                 }
 315         }
 316 }
 317
 318 static int nf_tables_register_hook(struct net *net,
 319                                    const struct nft_table *table,
 320                                    struct nft_chain *chain)
 321 {
 322         struct nft_base_chain *basechain;
 323         const struct nf_hook_ops *ops;
 324
 325         if (table->flags & NFT_TABLE_F_DORMANT ||
 326             !nft_is_base_chain(chain))
 327                 return 0;
 328
 329         basechain = nft_base_chain(chain);
 330         ops = &basechain->ops;
 331
 332         if (basechain->type->ops_register)
 333                 return basechain->type->ops_register(net, ops);
 334
 335         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 336                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 337
 338         return nf_register_net_hook(net, &basechain->ops);
 339 }
 340
 341 static void __nf_tables_unregister_hook(struct net *net,
 342                                         const struct nft_table *table,
 343                                         struct nft_chain *chain,
 344                                         bool release_netdev)
 345 {
 346         struct nft_base_chain *basechain;
 347         const struct nf_hook_ops *ops;
 348
 349         if (table->flags & NFT_TABLE_F_DORMANT ||
 350             !nft_is_base_chain(chain))
 351                 return;
 352         basechain = nft_base_chain(chain);
 353         ops = &basechain->ops;
 354
 355         if (basechain->type->ops_unregister)
 356                 return basechain->type->ops_unregister(net, ops);
 357
 358         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 359                 nft_netdev_unregister_hooks(net, &basechain->hook_list,
 360                                             release_netdev);
 361         else
 362                 nf_unregister_net_hook(net, &basechain->ops);
 363 }
 364
 365 static void nf_tables_unregister_hook(struct net *net,
 366                                       const struct nft_table *table,
 367                                       struct nft_chain *chain)
 368 {
 369         return __nf_tables_unregister_hook(net, table, chain, false);
 370 }
 371
 372 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 373 {
 374         struct nftables_pernet *nft_net = nft_pernet(net);
 375
 376         switch (trans->msg_type) {
 377         case NFT_MSG_NEWSET:
 378                 if (!nft_trans_set_update(trans) &&
 379                     nft_set_is_anonymous(nft_trans_set(trans)))
 380                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 381                 break;
 382         case NFT_MSG_NEWCHAIN:
 383                 if (!nft_trans_chain_update(trans) &&
 384                     nft_chain_binding(nft_trans_chain(trans)))
 385                         list_add_tail(&trans->binding_list, &nft_net->binding_list);
 386                 break;
 387         }
 388
 389         list_add_tail(&trans->list, &nft_net->commit_list);
 390 }
 391
 392 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 393 {
 394         struct nft_trans *trans;
 395
 396         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 397         if (trans == NULL)
 398                 return -ENOMEM;
 399
 400         if (msg_type == NFT_MSG_NEWTABLE)
 401                 nft_activate_next(ctx->net, ctx->table);
 402
 403         nft_trans_commit_list_add_tail(ctx->net, trans);
 404         return 0;
 405 }
 406
 407 static int nft_deltable(struct nft_ctx *ctx)
 408 {
 409         int err;
 410
 411         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 412         if (err < 0)
 413                 return err;
 414
 415         nft_deactivate_next(ctx->net, ctx->table);
 416         return err;
 417 }
 418
 419 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 420 {
 421         struct nft_trans *trans;
 422
 423         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 424         if (trans == NULL)
 425                 return ERR_PTR(-ENOMEM);
 426
 427         if (msg_type == NFT_MSG_NEWCHAIN) {
 428                 nft_activate_next(ctx->net, ctx->chain);
 429
 430                 if (ctx->nla[NFTA_CHAIN_ID]) {
 431                         nft_trans_chain_id(trans) =
 432                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 433                 }
 434         }
 435         nft_trans_chain(trans) = ctx->chain;
 436         nft_trans_commit_list_add_tail(ctx->net, trans);
 437
 438         return trans;
 439 }
 440
 441 static int nft_delchain(struct nft_ctx *ctx)
 442 {
 443         struct nft_trans *trans;
 444
 445         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 446         if (IS_ERR(trans))
 447                 return PTR_ERR(trans);
 448
 449         nft_use_dec(&ctx->table->use);
 450         nft_deactivate_next(ctx->net, ctx->chain);
 451
 452         return 0;
 453 }
 454
 455 void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
 456 {
 457         struct nft_expr *expr;
 458
 459         expr = nft_expr_first(rule);
 460         while (nft_expr_more(rule, expr)) {
 461                 if (expr->ops->activate)
 462                         expr->ops->activate(ctx, expr);
 463
 464                 expr = nft_expr_next(expr);
 465         }
 466 }
 467
 468 void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
 469                               enum nft_trans_phase phase)
 470 {
 471         struct nft_expr *expr;
 472
 473         expr = nft_expr_first(rule);
 474         while (nft_expr_more(rule, expr)) {
 475                 if (expr->ops->deactivate)
 476                         expr->ops->deactivate(ctx, expr, phase);
 477
 478                 expr = nft_expr_next(expr);
 479         }
 480 }
 481
 482 static int
 483 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 484 {
 485         /* You cannot delete the same rule twice */
 486         if (nft_is_active_next(ctx->net, rule)) {
 487                 nft_deactivate_next(ctx->net, rule);
 488                 nft_use_dec(&ctx->chain->use);
 489                 return 0;
 490         }
 491         return -ENOENT;
 492 }
 493
 494 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 495                                             struct nft_rule *rule)
 496 {
 497         struct nft_trans *trans;
 498
 499         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 500         if (trans == NULL)
 501                 return NULL;
 502
 503         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 504                 nft_trans_rule_id(trans) =
 505                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 506         }
 507         nft_trans_rule(trans) = rule;
 508         nft_trans_commit_list_add_tail(ctx->net, trans);
 509
 510         return trans;
 511 }
 512
 513 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 514 {
 515         struct nft_flow_rule *flow;
 516         struct nft_trans *trans;
 517         int err;
 518
 519         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 520         if (trans == NULL)
 521                 return -ENOMEM;
 522
 523         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 524                 flow = nft_flow_rule_create(ctx->net, rule);
 525                 if (IS_ERR(flow)) {
 526                         nft_trans_destroy(trans);
 527                         return PTR_ERR(flow);
 528                 }
 529
 530                 nft_trans_flow_rule(trans) = flow;
 531         }
 532
 533         err = nf_tables_delrule_deactivate(ctx, rule);
 534         if (err < 0) {
 535                 nft_trans_destroy(trans);
 536                 return err;
 537         }
 538         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 539
 540         return 0;
 541 }
 542
 543 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 544 {
 545         struct nft_rule *rule;
 546         int err;
 547
 548         list_for_each_entry(rule, &ctx->chain->rules, list) {
 549                 if (!nft_is_active_next(ctx->net, rule))
 550                         continue;
 551
 552                 err = nft_delrule(ctx, rule);
 553                 if (err < 0)
 554                         return err;
 555         }
 556         return 0;
 557 }
 558
 559 static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 560                                struct nft_set *set,
 561                                const struct nft_set_desc *desc)
 562 {
 563         struct nft_trans *trans;
 564
 565         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 566         if (trans == NULL)
 567                 return -ENOMEM;
 568
 569         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
 570                 nft_trans_set_id(trans) =
 571                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 572                 nft_activate_next(ctx->net, set);
 573         }
 574         nft_trans_set(trans) = set;
 575         if (desc) {
 576                 nft_trans_set_update(trans) = true;
 577                 nft_trans_set_gc_int(trans) = desc->gc_int;
 578                 nft_trans_set_timeout(trans) = desc->timeout;
 579                 nft_trans_set_size(trans) = desc->size;
 580         }
 581         nft_trans_commit_list_add_tail(ctx->net, trans);
 582
 583         return 0;
 584 }
 585
 586 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 587                              struct nft_set *set)
 588 {
 589         return __nft_trans_set_add(ctx, msg_type, set, NULL);
 590 }
 591
 592 static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
 593                                   struct nft_set *set,
 594                                   const struct nft_set_iter *iter,
 595                                   struct nft_elem_priv *elem_priv)
 596 {
 597         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 598
 599         if (!nft_set_elem_active(ext, iter->genmask))
 600                 return 0;
 601
 602         nft_set_elem_change_active(ctx->net, set, ext);
 603         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
 604
 605         return 0;
 606 }
 607
 608 struct nft_set_elem_catchall {
 609         struct list_head        list;
 610         struct rcu_head         rcu;
 611         struct nft_elem_priv    *elem;
 612 };
 613
 614 static void nft_map_catchall_deactivate(const struct nft_ctx *ctx,
 615                                         struct nft_set *set)
 616 {
 617         u8 genmask = nft_genmask_next(ctx->net);
 618         struct nft_set_elem_catchall *catchall;
 619         struct nft_set_ext *ext;
 620
 621         list_for_each_entry(catchall, &set->catchall_list, list) {
 622                 ext = nft_set_elem_ext(set, catchall->elem);
 623                 if (!nft_set_elem_active(ext, genmask))
 624                         continue;
 625
 626                 nft_set_elem_change_active(ctx->net, set, ext);
 627                 nft_setelem_data_deactivate(ctx->net, set, catchall->elem);
 628                 break;
 629         }
 630 }
 631
 632 static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set)
 633 {
 634         struct nft_set_iter iter = {
 635                 .genmask        = nft_genmask_next(ctx->net),
 636                 .type           = NFT_ITER_UPDATE,
 637                 .fn             = nft_mapelem_deactivate,
 638         };
 639
 640         set->ops->walk(ctx, set, &iter);
 641         WARN_ON_ONCE(iter.err);
 642
 643         nft_map_catchall_deactivate(ctx, set);
 644 }
 645
 646 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 647 {
 648         int err;
 649
 650         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 651         if (err < 0)
 652                 return err;
 653
 654         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 655                 nft_map_deactivate(ctx, set);
 656
 657         nft_deactivate_next(ctx->net, set);
 658         nft_use_dec(&ctx->table->use);
 659
 660         return err;
 661 }
 662
 663 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 664                              struct nft_object *obj)
 665 {
 666         struct nft_trans *trans;
 667
 668         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 669         if (trans == NULL)
 670                 return -ENOMEM;
 671
 672         if (msg_type == NFT_MSG_NEWOBJ)
 673                 nft_activate_next(ctx->net, obj);
 674
 675         nft_trans_obj(trans) = obj;
 676         nft_trans_commit_list_add_tail(ctx->net, trans);
 677
 678         return 0;
 679 }
 680
 681 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 682 {
 683         int err;
 684
 685         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 686         if (err < 0)
 687                 return err;
 688
 689         nft_deactivate_next(ctx->net, obj);
 690         nft_use_dec(&ctx->table->use);
 691
 692         return err;
 693 }
 694
 695 static struct nft_trans *
 696 nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 697                         struct nft_flowtable *flowtable)
 698 {
 699         struct nft_trans *trans;
 700
 701         trans = nft_trans_alloc(ctx, msg_type,
 702                                 sizeof(struct nft_trans_flowtable));
 703         if (trans == NULL)
 704                 return ERR_PTR(-ENOMEM);
 705
 706         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 707                 nft_activate_next(ctx->net, flowtable);
 708
 709         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 710         nft_trans_flowtable(trans) = flowtable;
 711         nft_trans_commit_list_add_tail(ctx->net, trans);
 712
 713         return trans;
 714 }
 715
 716 static int nft_delflowtable(struct nft_ctx *ctx,
 717                             struct nft_flowtable *flowtable)
 718 {
 719         struct nft_trans *trans;
 720
 721         trans = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 722         if (IS_ERR(trans))
 723                 return PTR_ERR(trans);
 724
 725         nft_deactivate_next(ctx->net, flowtable);
 726         nft_use_dec(&ctx->table->use);
 727
 728         return 0;
 729 }
 730
 731 static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
 732 {
 733         int i;
 734
 735         for (i = track->regs[dreg].num_reg; i > 0; i--)
 736                 __nft_reg_track_cancel(track, dreg - i);
 737 }
 738
 739 static void __nft_reg_track_update(struct nft_regs_track *track,
 740                                    const struct nft_expr *expr,
 741                                    u8 dreg, u8 num_reg)
 742 {
 743         track->regs[dreg].selector = expr;
 744         track->regs[dreg].bitwise = NULL;
 745         track->regs[dreg].num_reg = num_reg;
 746 }
 747
 748 void nft_reg_track_update(struct nft_regs_track *track,
 749                           const struct nft_expr *expr, u8 dreg, u8 len)
 750 {
 751         unsigned int regcount;
 752         int i;
 753
 754         __nft_reg_track_clobber(track, dreg);
 755
 756         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 757         for (i = 0; i < regcount; i++, dreg++)
 758                 __nft_reg_track_update(track, expr, dreg, i);
 759 }
 760 EXPORT_SYMBOL_GPL(nft_reg_track_update);
 761
 762 void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
 763 {
 764         unsigned int regcount;
 765         int i;
 766
 767         __nft_reg_track_clobber(track, dreg);
 768
 769         regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
 770         for (i = 0; i < regcount; i++, dreg++)
 771                 __nft_reg_track_cancel(track, dreg);
 772 }
 773 EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
 774
 775 void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
 776 {
 777         track->regs[dreg].selector = NULL;
 778         track->regs[dreg].bitwise = NULL;
 779         track->regs[dreg].num_reg = 0;
 780 }
 781 EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
 782
 783 /*
 784  * Tables
 785  */
 786
 787 static struct nft_table *nft_table_lookup(const struct net *net,
 788                                           const struct nlattr *nla,
 789                                           u8 family, u8 genmask, u32 nlpid)
 790 {
 791         struct nftables_pernet *nft_net;
 792         struct nft_table *table;
 793
 794         if (nla == NULL)
 795                 return ERR_PTR(-EINVAL);
 796
 797         nft_net = nft_pernet(net);
 798         list_for_each_entry_rcu(table, &nft_net->tables, list,
 799                                 lockdep_is_held(&nft_net->commit_mutex)) {
 800                 if (!nla_strcmp(nla, table->name) &&
 801                     table->family == family &&
 802                     nft_active_genmask(table, genmask)) {
 803                         if (nft_table_has_owner(table) &&
 804                             nlpid && table->nlpid != nlpid)
 805                                 return ERR_PTR(-EPERM);
 806
 807                         return table;
 808                 }
 809         }
 810
 811         return ERR_PTR(-ENOENT);
 812 }
 813
 814 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 815                                                    const struct nlattr *nla,
 816                                                    int family, u8 genmask, u32 nlpid)
 817 {
 818         struct nftables_pernet *nft_net;
 819         struct nft_table *table;
 820
 821         nft_net = nft_pernet(net);
 822         list_for_each_entry(table, &nft_net->tables, list) {
 823                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 824                     table->family == family &&
 825                     nft_active_genmask(table, genmask)) {
 826                         if (nft_table_has_owner(table) &&
 827                             nlpid && table->nlpid != nlpid)
 828                                 return ERR_PTR(-EPERM);
 829
 830                         return table;
 831                 }
 832         }
 833
 834         return ERR_PTR(-ENOENT);
 835 }
 836
 837 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 838 {
 839         return ++table->hgenerator;
 840 }
 841
 842 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 843
 844 static const struct nft_chain_type *
 845 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 846 {
 847         if (family >= NFPROTO_NUMPROTO ||
 848             type >= NFT_CHAIN_T_MAX)
 849                 return NULL;
 850
 851         return chain_type[family][type];
 852 }
 853
 854 static const struct nft_chain_type *
 855 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 856 {
 857         const struct nft_chain_type *type;
 858         int i;
 859
 860         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 861                 type = __nft_chain_type_get(family, i);
 862                 if (!type)
 863                         continue;
 864                 if (!nla_strcmp(nla, type->name))
 865                         return type;
 866         }
 867         return NULL;
 868 }
 869
 870 struct nft_module_request {
 871         struct list_head        list;
 872         char                    module[MODULE_NAME_LEN];
 873         bool                    done;
 874 };
 875
 876 #ifdef CONFIG_MODULES
 877 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 878                                       ...)
 879 {
 880         char module_name[MODULE_NAME_LEN];
 881         struct nftables_pernet *nft_net;
 882         struct nft_module_request *req;
 883         va_list args;
 884         int ret;
 885
 886         va_start(args, fmt);
 887         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 888         va_end(args);
 889         if (ret >= MODULE_NAME_LEN)
 890                 return 0;
 891
 892         nft_net = nft_pernet(net);
 893         list_for_each_entry(req, &nft_net->module_list, list) {
 894                 if (!strcmp(req->module, module_name)) {
 895                         if (req->done)
 896                                 return 0;
 897
 898                         /* A request to load this module already exists. */
 899                         return -EAGAIN;
 900                 }
 901         }
 902
 903         req = kmalloc(sizeof(*req), GFP_KERNEL);
 904         if (!req)
 905                 return -ENOMEM;
 906
 907         req->done = false;
 908         strscpy(req->module, module_name, MODULE_NAME_LEN);
 909         list_add_tail(&req->list, &nft_net->module_list);
 910
 911         return -EAGAIN;
 912 }
 913 EXPORT_SYMBOL_GPL(nft_request_module);
 914 #endif
 915
 916 static void lockdep_nfnl_nft_mutex_not_held(void)
 917 {
 918 #ifdef CONFIG_PROVE_LOCKING
 919         if (debug_locks)
 920                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 921 #endif
 922 }
 923
 924 static const struct nft_chain_type *
 925 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 926                             u8 family, bool autoload)
 927 {
 928         const struct nft_chain_type *type;
 929
 930         type = __nf_tables_chain_type_lookup(nla, family);
 931         if (type != NULL)
 932                 return type;
 933
 934         lockdep_nfnl_nft_mutex_not_held();
 935 #ifdef CONFIG_MODULES
 936         if (autoload) {
 937                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 938                                        nla_len(nla),
 939                                        (const char *)nla_data(nla)) == -EAGAIN)
 940                         return ERR_PTR(-EAGAIN);
 941         }
 942 #endif
 943         return ERR_PTR(-ENOENT);
 944 }
 945
 946 static __be16 nft_base_seq(const struct net *net)
 947 {
 948         struct nftables_pernet *nft_net = nft_pernet(net);
 949
 950         return htons(nft_net->base_seq & 0xffff);
 951 }
 952
 953 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 954         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 955                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 956         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 957         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 958         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 959                                     .len = NFT_USERDATA_MAXLEN }
 960 };
 961
 962 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 963                                      u32 portid, u32 seq, int event, u32 flags,
 964                                      int family, const struct nft_table *table)
 965 {
 966         struct nlmsghdr *nlh;
 967
 968         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 969         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 970                            NFNETLINK_V0, nft_base_seq(net));
 971         if (!nlh)
 972                 goto nla_put_failure;
 973
 974         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 975             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 976             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 977                          NFTA_TABLE_PAD))
 978                 goto nla_put_failure;
 979
 980         if (event == NFT_MSG_DELTABLE) {
 981                 nlmsg_end(skb, nlh);
 982                 return 0;
 983         }
 984
 985         if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 986                          htonl(table->flags & NFT_TABLE_F_MASK)))
 987                 goto nla_put_failure;
 988
 989         if (nft_table_has_owner(table) &&
 990             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 991                 goto nla_put_failure;
 992
 993         if (table->udata) {
 994                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 995                         goto nla_put_failure;
 996         }
 997
 998         nlmsg_end(skb, nlh);
 999         return 0;
1000
1001 nla_put_failure:
1002         nlmsg_trim(skb, nlh);
1003         return -1;
1004 }
1005
1006 struct nftnl_skb_parms {
1007         bool report;
1008 };
1009 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
1010
1011 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
1012                                struct list_head *notify_list)
1013 {
1014         NFT_CB(skb).report = report;
1015         list_add_tail(&skb->list, notify_list);
1016 }
1017
1018 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
1019 {
1020         struct nftables_pernet *nft_net;
1021         struct sk_buff *skb;
1022         u16 flags = 0;
1023         int err;
1024
1025         if (!ctx->report &&
1026             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1027                 return;
1028
1029         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1030         if (skb == NULL)
1031                 goto err;
1032
1033         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1034                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1035
1036         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
1037                                         event, flags, ctx->family, ctx->table);
1038         if (err < 0) {
1039                 kfree_skb(skb);
1040                 goto err;
1041         }
1042
1043         nft_net = nft_pernet(ctx->net);
1044         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1045         return;
1046 err:
1047         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1048 }
1049
1050 static int nf_tables_dump_tables(struct sk_buff *skb,
1051                                  struct netlink_callback *cb)
1052 {
1053         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1054         struct nftables_pernet *nft_net;
1055         const struct nft_table *table;
1056         unsigned int idx = 0, s_idx = cb->args[0];
1057         struct net *net = sock_net(skb->sk);
1058         int family = nfmsg->nfgen_family;
1059
1060         rcu_read_lock();
1061         nft_net = nft_pernet(net);
1062         cb->seq = READ_ONCE(nft_net->base_seq);
1063
1064         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1065                 if (family != NFPROTO_UNSPEC && family != table->family)
1066                         continue;
1067
1068                 if (idx < s_idx)
1069                         goto cont;
1070                 if (idx > s_idx)
1071                         memset(&cb->args[1], 0,
1072                                sizeof(cb->args) - sizeof(cb->args[0]));
1073                 if (!nft_is_active(net, table))
1074                         continue;
1075                 if (nf_tables_fill_table_info(skb, net,
1076                                               NETLINK_CB(cb->skb).portid,
1077                                               cb->nlh->nlmsg_seq,
1078                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
1079                                               table->family, table) < 0)
1080                         goto done;
1081
1082                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1083 cont:
1084                 idx++;
1085         }
1086 done:
1087         rcu_read_unlock();
1088         cb->args[0] = idx;
1089         return skb->len;
1090 }
1091
1092 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
1093                                       const struct nlmsghdr *nlh,
1094                                       struct netlink_dump_control *c)
1095 {
1096         int err;
1097
1098         if (!try_module_get(THIS_MODULE))
1099                 return -EINVAL;
1100
1101         rcu_read_unlock();
1102         err = netlink_dump_start(nlsk, skb, nlh, c);
1103         rcu_read_lock();
1104         module_put(THIS_MODULE);
1105
1106         return err;
1107 }
1108
1109 /* called with rcu_read_lock held */
1110 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
1111                               const struct nlattr * const nla[])
1112 {
1113         struct netlink_ext_ack *extack = info->extack;
1114         u8 genmask = nft_genmask_cur(info->net);
1115         u8 family = info->nfmsg->nfgen_family;
1116         const struct nft_table *table;
1117         struct net *net = info->net;
1118         struct sk_buff *skb2;
1119         int err;
1120
1121         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1122                 struct netlink_dump_control c = {
1123                         .dump = nf_tables_dump_tables,
1124                         .module = THIS_MODULE,
1125                 };
1126
1127                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1128         }
1129
1130         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
1131         if (IS_ERR(table)) {
1132                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
1133                 return PTR_ERR(table);
1134         }
1135
1136         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1137         if (!skb2)
1138                 return -ENOMEM;
1139
1140         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
1141                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
1142                                         0, family, table);
1143         if (err < 0)
1144                 goto err_fill_table_info;
1145
1146         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1147
1148 err_fill_table_info:
1149         kfree_skb(skb2);
1150         return err;
1151 }
1152
1153 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
1154 {
1155         struct nft_chain *chain;
1156         u32 i = 0;
1157
1158         list_for_each_entry(chain, &table->chains, list) {
1159                 if (!nft_is_active_next(net, chain))
1160                         continue;
1161                 if (!nft_is_base_chain(chain))
1162                         continue;
1163
1164                 if (cnt && i++ == cnt)
1165                         break;
1166
1167                 nf_tables_unregister_hook(net, table, chain);
1168         }
1169 }
1170
1171 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
1172 {
1173         struct nft_chain *chain;
1174         int err, i = 0;
1175
1176         list_for_each_entry(chain, &table->chains, list) {
1177                 if (!nft_is_active_next(net, chain))
1178                         continue;
1179                 if (!nft_is_base_chain(chain))
1180                         continue;
1181
1182                 err = nf_tables_register_hook(net, table, chain);
1183                 if (err < 0)
1184                         goto err_register_hooks;
1185
1186                 i++;
1187         }
1188         return 0;
1189
1190 err_register_hooks:
1191         if (i)
1192                 nft_table_disable(net, table, i);
1193         return err;
1194 }
1195
1196 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
1197 {
1198         table->flags &= ~NFT_TABLE_F_DORMANT;
1199         nft_table_disable(net, table, 0);
1200         table->flags |= NFT_TABLE_F_DORMANT;
1201 }
1202
1203 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
1204 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
1205 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
1206 #define __NFT_TABLE_F_WAS_ORPHAN        (__NFT_TABLE_F_INTERNAL << 2)
1207 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
1208                                          __NFT_TABLE_F_WAS_AWAKEN | \
1209                                          __NFT_TABLE_F_WAS_ORPHAN)
1210
1211 static bool nft_table_pending_update(const struct nft_ctx *ctx)
1212 {
1213         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1214         struct nft_trans *trans;
1215
1216         if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
1217                 return true;
1218
1219         list_for_each_entry(trans, &nft_net->commit_list, list) {
1220                 if (trans->ctx.table == ctx->table &&
1221                     ((trans->msg_type == NFT_MSG_NEWCHAIN &&
1222                       nft_trans_chain_update(trans)) ||
1223                      (trans->msg_type == NFT_MSG_DELCHAIN &&
1224                       nft_is_base_chain(trans->ctx.chain))))
1225                         return true;
1226         }
1227
1228         return false;
1229 }
1230
1231 static int nf_tables_updtable(struct nft_ctx *ctx)
1232 {
1233         struct nft_trans *trans;
1234         u32 flags;
1235         int ret;
1236
1237         if (!ctx->nla[NFTA_TABLE_FLAGS])
1238                 return 0;
1239
1240         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
1241         if (flags & ~NFT_TABLE_F_MASK)
1242                 return -EOPNOTSUPP;
1243
1244         if (flags == (ctx->table->flags & NFT_TABLE_F_MASK))
1245                 return 0;
1246
1247         if ((nft_table_has_owner(ctx->table) &&
1248              !(flags & NFT_TABLE_F_OWNER)) ||
1249             (flags & NFT_TABLE_F_OWNER &&
1250              !nft_table_is_orphan(ctx->table)))
1251                 return -EOPNOTSUPP;
1252
1253         if ((flags ^ ctx->table->flags) & NFT_TABLE_F_PERSIST)
1254                 return -EOPNOTSUPP;
1255
1256         /* No dormant off/on/off/on games in single transaction */
1257         if (nft_table_pending_update(ctx))
1258                 return -EINVAL;
1259
1260         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
1261                                 sizeof(struct nft_trans_table));
1262         if (trans == NULL)
1263                 return -ENOMEM;
1264
1265         if ((flags & NFT_TABLE_F_DORMANT) &&
1266             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
1267                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
1268                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
1269                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
1270         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
1271                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
1272                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
1273                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
1274                         ret = nf_tables_table_enable(ctx->net, ctx->table);
1275                         if (ret < 0)
1276                                 goto err_register_hooks;
1277
1278                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1279                 }
1280         }
1281
1282         if ((flags & NFT_TABLE_F_OWNER) &&
1283             !nft_table_has_owner(ctx->table)) {
1284                 ctx->table->nlpid = ctx->portid;
1285                 ctx->table->flags |= NFT_TABLE_F_OWNER |
1286                                      __NFT_TABLE_F_WAS_ORPHAN;
1287         }
1288
1289         nft_trans_table_update(trans) = true;
1290         nft_trans_commit_list_add_tail(ctx->net, trans);
1291
1292         return 0;
1293
1294 err_register_hooks:
1295         ctx->table->flags |= NFT_TABLE_F_DORMANT;
1296         nft_trans_destroy(trans);
1297         return ret;
1298 }
1299
1300 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1301 {
1302         const char *name = data;
1303
1304         return jhash(name, strlen(name), seed);
1305 }
1306
1307 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1308 {
1309         const struct nft_chain *chain = data;
1310
1311         return nft_chain_hash(chain->name, 0, seed);
1312 }
1313
1314 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1315                               const void *ptr)
1316 {
1317         const struct nft_chain *chain = ptr;
1318         const char *name = arg->key;
1319
1320         return strcmp(chain->name, name);
1321 }
1322
1323 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1324 {
1325         const struct nft_object_hash_key *k = data;
1326
1327         seed ^= hash_ptr(k->table, 32);
1328
1329         return jhash(k->name, strlen(k->name), seed);
1330 }
1331
1332 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1333 {
1334         const struct nft_object *obj = data;
1335
1336         return nft_objname_hash(&obj->key, 0, seed);
1337 }
1338
1339 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1340                                 const void *ptr)
1341 {
1342         const struct nft_object_hash_key *k = arg->key;
1343         const struct nft_object *obj = ptr;
1344
1345         if (obj->key.table != k->table)
1346                 return -1;
1347
1348         return strcmp(obj->key.name, k->name);
1349 }
1350
1351 static bool nft_supported_family(u8 family)
1352 {
1353         return false
1354 #ifdef CONFIG_NF_TABLES_INET
1355                 || family == NFPROTO_INET
1356 #endif
1357 #ifdef CONFIG_NF_TABLES_IPV4
1358                 || family == NFPROTO_IPV4
1359 #endif
1360 #ifdef CONFIG_NF_TABLES_ARP
1361                 || family == NFPROTO_ARP
1362 #endif
1363 #ifdef CONFIG_NF_TABLES_NETDEV
1364                 || family == NFPROTO_NETDEV
1365 #endif
1366 #if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
1367                 || family == NFPROTO_BRIDGE
1368 #endif
1369 #ifdef CONFIG_NF_TABLES_IPV6
1370                 || family == NFPROTO_IPV6
1371 #endif
1372                 ;
1373 }
1374
1375 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1376                               const struct nlattr * const nla[])
1377 {
1378         struct nftables_pernet *nft_net = nft_pernet(info->net);
1379         struct netlink_ext_ack *extack = info->extack;
1380         u8 genmask = nft_genmask_next(info->net);
1381         u8 family = info->nfmsg->nfgen_family;
1382         struct net *net = info->net;
1383         const struct nlattr *attr;
1384         struct nft_table *table;
1385         struct nft_ctx ctx;
1386         u32 flags = 0;
1387         int err;
1388
1389         if (!nft_supported_family(family))
1390                 return -EOPNOTSUPP;
1391
1392         lockdep_assert_held(&nft_net->commit_mutex);
1393         attr = nla[NFTA_TABLE_NAME];
1394         table = nft_table_lookup(net, attr, family, genmask,
1395                                  NETLINK_CB(skb).portid);
1396         if (IS_ERR(table)) {
1397                 if (PTR_ERR(table) != -ENOENT)
1398                         return PTR_ERR(table);
1399         } else {
1400                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1401                         NL_SET_BAD_ATTR(extack, attr);
1402                         return -EEXIST;
1403                 }
1404                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1405                         return -EOPNOTSUPP;
1406
1407                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1408
1409                 return nf_tables_updtable(&ctx);
1410         }
1411
1412         if (nla[NFTA_TABLE_FLAGS]) {
1413                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1414                 if (flags & ~NFT_TABLE_F_MASK)
1415                         return -EOPNOTSUPP;
1416         }
1417
1418         err = -ENOMEM;
1419         table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
1420         if (table == NULL)
1421                 goto err_kzalloc;
1422
1423         table->validate_state = nft_net->validate_state;
1424         table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
1425         if (table->name == NULL)
1426                 goto err_strdup;
1427
1428         if (nla[NFTA_TABLE_USERDATA]) {
1429                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
1430                 if (table->udata == NULL)
1431                         goto err_table_udata;
1432
1433                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1434         }
1435
1436         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1437         if (err)
1438                 goto err_chain_ht;
1439
1440         INIT_LIST_HEAD(&table->chains);
1441         INIT_LIST_HEAD(&table->sets);
1442         INIT_LIST_HEAD(&table->objects);
1443         INIT_LIST_HEAD(&table->flowtables);
1444         table->family = family;
1445         table->flags = flags;
1446         table->handle = ++nft_net->table_handle;
1447         if (table->flags & NFT_TABLE_F_OWNER)
1448                 table->nlpid = NETLINK_CB(skb).portid;
1449
1450         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1451         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1452         if (err < 0)
1453                 goto err_trans;
1454
1455         list_add_tail_rcu(&table->list, &nft_net->tables);
1456         return 0;
1457 err_trans:
1458         rhltable_destroy(&table->chains_ht);
1459 err_chain_ht:
1460         kfree(table->udata);
1461 err_table_udata:
1462         kfree(table->name);
1463 err_strdup:
1464         kfree(table);
1465 err_kzalloc:
1466         return err;
1467 }
1468
1469 static int nft_flush_table(struct nft_ctx *ctx)
1470 {
1471         struct nft_flowtable *flowtable, *nft;
1472         struct nft_chain *chain, *nc;
1473         struct nft_object *obj, *ne;
1474         struct nft_set *set, *ns;
1475         int err;
1476
1477         list_for_each_entry(chain, &ctx->table->chains, list) {
1478                 if (!nft_is_active_next(ctx->net, chain))
1479                         continue;
1480
1481                 if (nft_chain_binding(chain))
1482                         continue;
1483
1484                 ctx->chain = chain;
1485
1486                 err = nft_delrule_by_chain(ctx);
1487                 if (err < 0)
1488                         goto out;
1489         }
1490
1491         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1492                 if (!nft_is_active_next(ctx->net, set))
1493                         continue;
1494
1495                 if (nft_set_is_anonymous(set))
1496                         continue;
1497
1498                 err = nft_delset(ctx, set);
1499                 if (err < 0)
1500                         goto out;
1501         }
1502
1503         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1504                 if (!nft_is_active_next(ctx->net, flowtable))
1505                         continue;
1506
1507                 err = nft_delflowtable(ctx, flowtable);
1508                 if (err < 0)
1509                         goto out;
1510         }
1511
1512         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1513                 if (!nft_is_active_next(ctx->net, obj))
1514                         continue;
1515
1516                 err = nft_delobj(ctx, obj);
1517                 if (err < 0)
1518                         goto out;
1519         }
1520
1521         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1522                 if (!nft_is_active_next(ctx->net, chain))
1523                         continue;
1524
1525                 if (nft_chain_binding(chain))
1526                         continue;
1527
1528                 ctx->chain = chain;
1529
1530                 err = nft_delchain(ctx);
1531                 if (err < 0)
1532                         goto out;
1533         }
1534
1535         err = nft_deltable(ctx);
1536 out:
1537         return err;
1538 }
1539
1540 static int nft_flush(struct nft_ctx *ctx, int family)
1541 {
1542         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1543         const struct nlattr * const *nla = ctx->nla;
1544         struct nft_table *table, *nt;
1545         int err = 0;
1546
1547         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1548                 if (family != AF_UNSPEC && table->family != family)
1549                         continue;
1550
1551                 ctx->family = table->family;
1552
1553                 if (!nft_is_active_next(ctx->net, table))
1554                         continue;
1555
1556                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1557                         continue;
1558
1559                 if (nla[NFTA_TABLE_NAME] &&
1560                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1561                         continue;
1562
1563                 ctx->table = table;
1564
1565                 err = nft_flush_table(ctx);
1566                 if (err < 0)
1567                         goto out;
1568         }
1569 out:
1570         return err;
1571 }
1572
1573 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1574                               const struct nlattr * const nla[])
1575 {
1576         struct netlink_ext_ack *extack = info->extack;
1577         u8 genmask = nft_genmask_next(info->net);
1578         u8 family = info->nfmsg->nfgen_family;
1579         struct net *net = info->net;
1580         const struct nlattr *attr;
1581         struct nft_table *table;
1582         struct nft_ctx ctx;
1583
1584         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1585         if (family == AF_UNSPEC ||
1586             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1587                 return nft_flush(&ctx, family);
1588
1589         if (nla[NFTA_TABLE_HANDLE]) {
1590                 attr = nla[NFTA_TABLE_HANDLE];
1591                 table = nft_table_lookup_byhandle(net, attr, family, genmask,
1592                                                   NETLINK_CB(skb).portid);
1593         } else {
1594                 attr = nla[NFTA_TABLE_NAME];
1595                 table = nft_table_lookup(net, attr, family, genmask,
1596                                          NETLINK_CB(skb).portid);
1597         }
1598
1599         if (IS_ERR(table)) {
1600                 if (PTR_ERR(table) == -ENOENT &&
1601                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
1602                         return 0;
1603
1604                 NL_SET_BAD_ATTR(extack, attr);
1605                 return PTR_ERR(table);
1606         }
1607
1608         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1609             table->use > 0)
1610                 return -EBUSY;
1611
1612         ctx.family = family;
1613         ctx.table = table;
1614
1615         return nft_flush_table(&ctx);
1616 }
1617
1618 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1619 {
1620         if (WARN_ON(ctx->table->use > 0))
1621                 return;
1622
1623         rhltable_destroy(&ctx->table->chains_ht);
1624         kfree(ctx->table->name);
1625         kfree(ctx->table->udata);
1626         kfree(ctx->table);
1627 }
1628
1629 void nft_register_chain_type(const struct nft_chain_type *ctype)
1630 {
1631         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1632         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1633                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1634                 return;
1635         }
1636         chain_type[ctype->family][ctype->type] = ctype;
1637         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1638 }
1639 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1640
1641 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1642 {
1643         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1644         chain_type[ctype->family][ctype->type] = NULL;
1645         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1646 }
1647 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1648
1649 /*
1650  * Chains
1651  */
1652
1653 static struct nft_chain *
1654 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1655 {
1656         struct nft_chain *chain;
1657
1658         list_for_each_entry(chain, &table->chains, list) {
1659                 if (chain->handle == handle &&
1660                     nft_active_genmask(chain, genmask))
1661                         return chain;
1662         }
1663
1664         return ERR_PTR(-ENOENT);
1665 }
1666
1667 static bool lockdep_commit_lock_is_held(const struct net *net)
1668 {
1669 #ifdef CONFIG_PROVE_LOCKING
1670         struct nftables_pernet *nft_net = nft_pernet(net);
1671
1672         return lockdep_is_held(&nft_net->commit_mutex);
1673 #else
1674         return true;
1675 #endif
1676 }
1677
1678 static struct nft_chain *nft_chain_lookup(struct net *net,
1679                                           struct nft_table *table,
1680                                           const struct nlattr *nla, u8 genmask)
1681 {
1682         char search[NFT_CHAIN_MAXNAMELEN + 1];
1683         struct rhlist_head *tmp, *list;
1684         struct nft_chain *chain;
1685
1686         if (nla == NULL)
1687                 return ERR_PTR(-EINVAL);
1688
1689         nla_strscpy(search, nla, sizeof(search));
1690
1691         WARN_ON(!rcu_read_lock_held() &&
1692                 !lockdep_commit_lock_is_held(net));
1693
1694         chain = ERR_PTR(-ENOENT);
1695         rcu_read_lock();
1696         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1697         if (!list)
1698                 goto out_unlock;
1699
1700         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1701                 if (nft_active_genmask(chain, genmask))
1702                         goto out_unlock;
1703         }
1704         chain = ERR_PTR(-ENOENT);
1705 out_unlock:
1706         rcu_read_unlock();
1707         return chain;
1708 }
1709
1710 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1711         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1712                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1713         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1714         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1715                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1716         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1717         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1718         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1719                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1720         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1721         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1722         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1723         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1724                                     .len = NFT_USERDATA_MAXLEN },
1725 };
1726
1727 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1728         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1729         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1730         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1731                                     .len = IFNAMSIZ - 1 },
1732 };
1733
1734 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1735 {
1736         struct nft_stats *cpu_stats, total;
1737         struct nlattr *nest;
1738         unsigned int seq;
1739         u64 pkts, bytes;
1740         int cpu;
1741
1742         if (!stats)
1743                 return 0;
1744
1745         memset(&total, 0, sizeof(total));
1746         for_each_possible_cpu(cpu) {
1747                 cpu_stats = per_cpu_ptr(stats, cpu);
1748                 do {
1749                         seq = u64_stats_fetch_begin(&cpu_stats->syncp);
1750                         pkts = cpu_stats->pkts;
1751                         bytes = cpu_stats->bytes;
1752                 } while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
1753                 total.pkts += pkts;
1754                 total.bytes += bytes;
1755         }
1756         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1757         if (nest == NULL)
1758                 goto nla_put_failure;
1759
1760         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1761                          NFTA_COUNTER_PAD) ||
1762             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1763                          NFTA_COUNTER_PAD))
1764                 goto nla_put_failure;
1765
1766         nla_nest_end(skb, nest);
1767         return 0;
1768
1769 nla_put_failure:
1770         return -ENOSPC;
1771 }
1772
1773 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1774                                    const struct nft_base_chain *basechain,
1775                                    const struct list_head *hook_list)
1776 {
1777         const struct nf_hook_ops *ops = &basechain->ops;
1778         struct nft_hook *hook, *first = NULL;
1779         struct nlattr *nest, *nest_devs;
1780         int n = 0;
1781
1782         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1783         if (nest == NULL)
1784                 goto nla_put_failure;
1785         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1786                 goto nla_put_failure;
1787         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1788                 goto nla_put_failure;
1789
1790         if (nft_base_chain_netdev(family, ops->hooknum)) {
1791                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1792                 if (!nest_devs)
1793                         goto nla_put_failure;
1794
1795                 if (!hook_list)
1796                         hook_list = &basechain->hook_list;
1797
1798                 list_for_each_entry(hook, hook_list, list) {
1799                         if (!first)
1800                                 first = hook;
1801
1802                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1803                                            hook->ops.dev->name))
1804                                 goto nla_put_failure;
1805                         n++;
1806                 }
1807                 nla_nest_end(skb, nest_devs);
1808
1809                 if (n == 1 &&
1810                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1811                         goto nla_put_failure;
1812         }
1813         nla_nest_end(skb, nest);
1814
1815         return 0;
1816 nla_put_failure:
1817         return -1;
1818 }
1819
1820 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1821                                      u32 portid, u32 seq, int event, u32 flags,
1822                                      int family, const struct nft_table *table,
1823                                      const struct nft_chain *chain,
1824                                      const struct list_head *hook_list)
1825 {
1826         struct nlmsghdr *nlh;
1827
1828         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1829         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1830                            NFNETLINK_V0, nft_base_seq(net));
1831         if (!nlh)
1832                 goto nla_put_failure;
1833
1834         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
1835             nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
1836             nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1837                          NFTA_CHAIN_PAD))
1838                 goto nla_put_failure;
1839
1840         if (event == NFT_MSG_DELCHAIN && !hook_list) {
1841                 nlmsg_end(skb, nlh);
1842                 return 0;
1843         }
1844
1845         if (nft_is_base_chain(chain)) {
1846                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1847                 struct nft_stats __percpu *stats;
1848
1849                 if (nft_dump_basechain_hook(skb, family, basechain, hook_list))
1850                         goto nla_put_failure;
1851
1852                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1853                                  htonl(basechain->policy)))
1854                         goto nla_put_failure;
1855
1856                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1857                         goto nla_put_failure;
1858
1859                 stats = rcu_dereference_check(basechain->stats,
1860                                               lockdep_commit_lock_is_held(net));
1861                 if (nft_dump_stats(skb, stats))
1862                         goto nla_put_failure;
1863         }
1864
1865         if (chain->flags &&
1866             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1867                 goto nla_put_failure;
1868
1869         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1870                 goto nla_put_failure;
1871
1872         if (chain->udata &&
1873             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1874                 goto nla_put_failure;
1875
1876         nlmsg_end(skb, nlh);
1877         return 0;
1878
1879 nla_put_failure:
1880         nlmsg_trim(skb, nlh);
1881         return -1;
1882 }
1883
1884 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
1885                                    const struct list_head *hook_list)
1886 {
1887         struct nftables_pernet *nft_net;
1888         struct sk_buff *skb;
1889         u16 flags = 0;
1890         int err;
1891
1892         if (!ctx->report &&
1893             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1894                 return;
1895
1896         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1897         if (skb == NULL)
1898                 goto err;
1899
1900         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
1901                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
1902
1903         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1904                                         event, flags, ctx->family, ctx->table,
1905                                         ctx->chain, hook_list);
1906         if (err < 0) {
1907                 kfree_skb(skb);
1908                 goto err;
1909         }
1910
1911         nft_net = nft_pernet(ctx->net);
1912         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1913         return;
1914 err:
1915         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1916 }
1917
1918 static int nf_tables_dump_chains(struct sk_buff *skb,
1919                                  struct netlink_callback *cb)
1920 {
1921         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1922         unsigned int idx = 0, s_idx = cb->args[0];
1923         struct net *net = sock_net(skb->sk);
1924         int family = nfmsg->nfgen_family;
1925         struct nftables_pernet *nft_net;
1926         const struct nft_table *table;
1927         const struct nft_chain *chain;
1928
1929         rcu_read_lock();
1930         nft_net = nft_pernet(net);
1931         cb->seq = READ_ONCE(nft_net->base_seq);
1932
1933         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1934                 if (family != NFPROTO_UNSPEC && family != table->family)
1935                         continue;
1936
1937                 list_for_each_entry_rcu(chain, &table->chains, list) {
1938                         if (idx < s_idx)
1939                                 goto cont;
1940                         if (idx > s_idx)
1941                                 memset(&cb->args[1], 0,
1942                                        sizeof(cb->args) - sizeof(cb->args[0]));
1943                         if (!nft_is_active(net, chain))
1944                                 continue;
1945                         if (nf_tables_fill_chain_info(skb, net,
1946                                                       NETLINK_CB(cb->skb).portid,
1947                                                       cb->nlh->nlmsg_seq,
1948                                                       NFT_MSG_NEWCHAIN,
1949                                                       NLM_F_MULTI,
1950                                                       table->family, table,
1951                                                       chain, NULL) < 0)
1952                                 goto done;
1953
1954                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1955 cont:
1956                         idx++;
1957                 }
1958         }
1959 done:
1960         rcu_read_unlock();
1961         cb->args[0] = idx;
1962         return skb->len;
1963 }
1964
1965 /* called with rcu_read_lock held */
1966 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1967                               const struct nlattr * const nla[])
1968 {
1969         struct netlink_ext_ack *extack = info->extack;
1970         u8 genmask = nft_genmask_cur(info->net);
1971         u8 family = info->nfmsg->nfgen_family;
1972         const struct nft_chain *chain;
1973         struct net *net = info->net;
1974         struct nft_table *table;
1975         struct sk_buff *skb2;
1976         int err;
1977
1978         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1979                 struct netlink_dump_control c = {
1980                         .dump = nf_tables_dump_chains,
1981                         .module = THIS_MODULE,
1982                 };
1983
1984                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1985         }
1986
1987         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1988         if (IS_ERR(table)) {
1989                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1990                 return PTR_ERR(table);
1991         }
1992
1993         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1994         if (IS_ERR(chain)) {
1995                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1996                 return PTR_ERR(chain);
1997         }
1998
1999         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
2000         if (!skb2)
2001                 return -ENOMEM;
2002
2003         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
2004                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
2005                                         0, family, table, chain, NULL);
2006         if (err < 0)
2007                 goto err_fill_chain_info;
2008
2009         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
2010
2011 err_fill_chain_info:
2012         kfree_skb(skb2);
2013         return err;
2014 }
2015
2016 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
2017         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
2018         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
2019 };
2020
2021 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
2022 {
2023         struct nlattr *tb[NFTA_COUNTER_MAX+1];
2024         struct nft_stats __percpu *newstats;
2025         struct nft_stats *stats;
2026         int err;
2027
2028         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
2029                                           nft_counter_policy, NULL);
2030         if (err < 0)
2031                 return ERR_PTR(err);
2032
2033         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
2034                 return ERR_PTR(-EINVAL);
2035
2036         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
2037         if (newstats == NULL)
2038                 return ERR_PTR(-ENOMEM);
2039
2040         /* Restore old counters on this cpu, no problem. Per-cpu statistics
2041          * are not exposed to userspace.
2042          */
2043         preempt_disable();
2044         stats = this_cpu_ptr(newstats);
2045         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
2046         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
2047         preempt_enable();
2048
2049         return newstats;
2050 }
2051
2052 static void nft_chain_stats_replace(struct nft_trans *trans)
2053 {
2054         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
2055
2056         if (!nft_trans_chain_stats(trans))
2057                 return;
2058
2059         nft_trans_chain_stats(trans) =
2060                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
2061                                     lockdep_commit_lock_is_held(trans->ctx.net));
2062
2063         if (!nft_trans_chain_stats(trans))
2064                 static_branch_inc(&nft_counters_enabled);
2065 }
2066
2067 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
2068 {
2069         struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
2070         struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
2071
2072         if (g0 != g1)
2073                 kvfree(g1);
2074         kvfree(g0);
2075
2076         /* should be NULL either via abort or via successful commit */
2077         WARN_ON_ONCE(chain->blob_next);
2078         kvfree(chain->blob_next);
2079 }
2080
2081 void nf_tables_chain_destroy(struct nft_ctx *ctx)
2082 {
2083         struct nft_chain *chain = ctx->chain;
2084         struct nft_hook *hook, *next;
2085
2086         if (WARN_ON(chain->use > 0))
2087                 return;
2088
2089         /* no concurrent access possible anymore */
2090         nf_tables_chain_free_chain_rules(chain);
2091
2092         if (nft_is_base_chain(chain)) {
2093                 struct nft_base_chain *basechain = nft_base_chain(chain);
2094
2095                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2096                         list_for_each_entry_safe(hook, next,
2097                                                  &basechain->hook_list, list) {
2098                                 list_del_rcu(&hook->list);
2099                                 kfree_rcu(hook, rcu);
2100                         }
2101                 }
2102                 module_put(basechain->type->owner);
2103                 if (rcu_access_pointer(basechain->stats)) {
2104                         static_branch_dec(&nft_counters_enabled);
2105                         free_percpu(rcu_dereference_raw(basechain->stats));
2106                 }
2107                 kfree(chain->name);
2108                 kfree(chain->udata);
2109                 kfree(basechain);
2110         } else {
2111                 kfree(chain->name);
2112                 kfree(chain->udata);
2113                 kfree(chain);
2114         }
2115 }
2116
2117 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
2118                                               const struct nlattr *attr)
2119 {
2120         struct net_device *dev;
2121         char ifname[IFNAMSIZ];
2122         struct nft_hook *hook;
2123         int err;
2124
2125         hook = kzalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
2126         if (!hook) {
2127                 err = -ENOMEM;
2128                 goto err_hook_alloc;
2129         }
2130
2131         nla_strscpy(ifname, attr, IFNAMSIZ);
2132         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
2133          * indirectly serializing all the other holders of the commit_mutex with
2134          * the rtnl_mutex.
2135          */
2136         dev = __dev_get_by_name(net, ifname);
2137         if (!dev) {
2138                 err = -ENOENT;
2139                 goto err_hook_dev;
2140         }
2141         hook->ops.dev = dev;
2142
2143         return hook;
2144
2145 err_hook_dev:
2146         kfree(hook);
2147 err_hook_alloc:
2148         return ERR_PTR(err);
2149 }
2150
2151 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
2152                                            const struct nft_hook *this)
2153 {
2154         struct nft_hook *hook;
2155
2156         list_for_each_entry(hook, hook_list, list) {
2157                 if (this->ops.dev == hook->ops.dev)
2158                         return hook;
2159         }
2160
2161         return NULL;
2162 }
2163
2164 static int nf_tables_parse_netdev_hooks(struct net *net,
2165                                         const struct nlattr *attr,
2166                                         struct list_head *hook_list,
2167                                         struct netlink_ext_ack *extack)
2168 {
2169         struct nft_hook *hook, *next;
2170         const struct nlattr *tmp;
2171         int rem, n = 0, err;
2172
2173         nla_for_each_nested(tmp, attr, rem) {
2174                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
2175                         err = -EINVAL;
2176                         goto err_hook;
2177                 }
2178
2179                 hook = nft_netdev_hook_alloc(net, tmp);
2180                 if (IS_ERR(hook)) {
2181                         NL_SET_BAD_ATTR(extack, tmp);
2182                         err = PTR_ERR(hook);
2183                         goto err_hook;
2184                 }
2185                 if (nft_hook_list_find(hook_list, hook)) {
2186                         NL_SET_BAD_ATTR(extack, tmp);
2187                         kfree(hook);
2188                         err = -EEXIST;
2189                         goto err_hook;
2190                 }
2191                 list_add_tail(&hook->list, hook_list);
2192                 n++;
2193
2194                 if (n == NFT_NETDEVICE_MAX) {
2195                         err = -EFBIG;
2196                         goto err_hook;
2197                 }
2198         }
2199
2200         return 0;
2201
2202 err_hook:
2203         list_for_each_entry_safe(hook, next, hook_list, list) {
2204                 list_del(&hook->list);
2205                 kfree(hook);
2206         }
2207         return err;
2208 }
2209
2210 struct nft_chain_hook {
2211         u32                             num;
2212         s32                             priority;
2213         const struct nft_chain_type     *type;
2214         struct list_head                list;
2215 };
2216
2217 static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
2218                                   struct list_head *hook_list,
2219                                   struct netlink_ext_ack *extack, u32 flags)
2220 {
2221         struct nft_hook *hook;
2222         int err;
2223
2224         if (tb[NFTA_HOOK_DEV]) {
2225                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
2226                 if (IS_ERR(hook)) {
2227                         NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
2228                         return PTR_ERR(hook);
2229                 }
2230
2231                 list_add_tail(&hook->list, hook_list);
2232         } else if (tb[NFTA_HOOK_DEVS]) {
2233                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
2234                                                    hook_list, extack);
2235                 if (err < 0)
2236                         return err;
2237
2238         }
2239
2240         if (flags & NFT_CHAIN_HW_OFFLOAD &&
2241             list_empty(hook_list))
2242                 return -EINVAL;
2243
2244         return 0;
2245 }
2246
2247 static int nft_chain_parse_hook(struct net *net,
2248                                 struct nft_base_chain *basechain,
2249                                 const struct nlattr * const nla[],
2250                                 struct nft_chain_hook *hook, u8 family,
2251                                 u32 flags, struct netlink_ext_ack *extack)
2252 {
2253         struct nftables_pernet *nft_net = nft_pernet(net);
2254         struct nlattr *ha[NFTA_HOOK_MAX + 1];
2255         const struct nft_chain_type *type;
2256         int err;
2257
2258         lockdep_assert_held(&nft_net->commit_mutex);
2259         lockdep_nfnl_nft_mutex_not_held();
2260
2261         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
2262                                           nla[NFTA_CHAIN_HOOK],
2263                                           nft_hook_policy, NULL);
2264         if (err < 0)
2265                 return err;
2266
2267         if (!basechain) {
2268                 if (!ha[NFTA_HOOK_HOOKNUM] ||
2269                     !ha[NFTA_HOOK_PRIORITY]) {
2270                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2271                         return -ENOENT;
2272                 }
2273
2274                 hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2275                 hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2276
2277                 type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
2278                 if (!type)
2279                         return -EOPNOTSUPP;
2280
2281                 if (nla[NFTA_CHAIN_TYPE]) {
2282                         type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
2283                                                            family, true);
2284                         if (IS_ERR(type)) {
2285                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2286                                 return PTR_ERR(type);
2287                         }
2288                 }
2289                 if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
2290                         return -EOPNOTSUPP;
2291
2292                 if (type->type == NFT_CHAIN_T_NAT &&
2293                     hook->priority <= NF_IP_PRI_CONNTRACK)
2294                         return -EOPNOTSUPP;
2295         } else {
2296                 if (ha[NFTA_HOOK_HOOKNUM]) {
2297                         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
2298                         if (hook->num != basechain->ops.hooknum)
2299                                 return -EOPNOTSUPP;
2300                 }
2301                 if (ha[NFTA_HOOK_PRIORITY]) {
2302                         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
2303                         if (hook->priority != basechain->ops.priority)
2304                                 return -EOPNOTSUPP;
2305                 }
2306
2307                 if (nla[NFTA_CHAIN_TYPE]) {
2308                         type = __nf_tables_chain_type_lookup(nla[NFTA_CHAIN_TYPE],
2309                                                              family);
2310                         if (!type) {
2311                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2312                                 return -ENOENT;
2313                         }
2314                 } else {
2315                         type = basechain->type;
2316                 }
2317         }
2318
2319         if (!try_module_get(type->owner)) {
2320                 if (nla[NFTA_CHAIN_TYPE])
2321                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
2322                 return -ENOENT;
2323         }
2324
2325         hook->type = type;
2326
2327         INIT_LIST_HEAD(&hook->list);
2328         if (nft_base_chain_netdev(family, hook->num)) {
2329                 err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
2330                 if (err < 0) {
2331                         module_put(type->owner);
2332                         return err;
2333                 }
2334         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
2335                 module_put(type->owner);
2336                 return -EOPNOTSUPP;
2337         }
2338
2339         return 0;
2340 }
2341
2342 static void nft_chain_release_hook(struct nft_chain_hook *hook)
2343 {
2344         struct nft_hook *h, *next;
2345
2346         list_for_each_entry_safe(h, next, &hook->list, list) {
2347                 list_del(&h->list);
2348                 kfree(h);
2349         }
2350         module_put(hook->type->owner);
2351 }
2352
2353 static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
2354 {
2355         struct nft_rule_dp_last *lrule;
2356
2357         BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
2358
2359         lrule = (struct nft_rule_dp_last *)ptr;
2360         lrule->end.is_last = 1;
2361         lrule->chain = chain;
2362         /* blob size does not include the trailer rule */
2363 }
2364
2365 static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
2366                                                          unsigned int size)
2367 {
2368         struct nft_rule_blob *blob;
2369
2370         if (size > INT_MAX)
2371                 return NULL;
2372
2373         size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
2374
2375         blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
2376         if (!blob)
2377                 return NULL;
2378
2379         blob->size = 0;
2380         nft_last_rule(chain, blob->data);
2381
2382         return blob;
2383 }
2384
2385 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2386                                     const struct nft_chain_hook *hook,
2387                                     struct nft_chain *chain)
2388 {
2389         ops->pf                 = family;
2390         ops->hooknum            = hook->num;
2391         ops->priority           = hook->priority;
2392         ops->priv               = chain;
2393         ops->hook               = hook->type->hooks[ops->hooknum];
2394         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2395 }
2396
2397 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2398                               struct nft_chain_hook *hook, u32 flags)
2399 {
2400         struct nft_chain *chain;
2401         struct nft_hook *h;
2402
2403         basechain->type = hook->type;
2404         INIT_LIST_HEAD(&basechain->hook_list);
2405         chain = &basechain->chain;
2406
2407         if (nft_base_chain_netdev(family, hook->num)) {
2408                 list_splice_init(&hook->list, &basechain->hook_list);
2409                 list_for_each_entry(h, &basechain->hook_list, list)
2410                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2411         }
2412         nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2413
2414         chain->flags |= NFT_CHAIN_BASE | flags;
2415         basechain->policy = NF_ACCEPT;
2416         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2417             !nft_chain_offload_support(basechain)) {
2418                 list_splice_init(&basechain->hook_list, &hook->list);
2419                 return -EOPNOTSUPP;
2420         }
2421
2422         flow_block_init(&basechain->flow_block);
2423
2424         return 0;
2425 }
2426
2427 int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2428 {
2429         int err;
2430
2431         err = rhltable_insert_key(&table->chains_ht, chain->name,
2432                                   &chain->rhlhead, nft_chain_ht_params);
2433         if (err)
2434                 return err;
2435
2436         list_add_tail_rcu(&chain->list, &table->chains);
2437
2438         return 0;
2439 }
2440
2441 static u64 chain_id;
2442
2443 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2444                               u8 policy, u32 flags,
2445                               struct netlink_ext_ack *extack)
2446 {
2447         const struct nlattr * const *nla = ctx->nla;
2448         struct nft_table *table = ctx->table;
2449         struct nft_base_chain *basechain;
2450         struct net *net = ctx->net;
2451         char name[NFT_NAME_MAXLEN];
2452         struct nft_rule_blob *blob;
2453         struct nft_trans *trans;
2454         struct nft_chain *chain;
2455         int err;
2456
2457         if (nla[NFTA_CHAIN_HOOK]) {
2458                 struct nft_stats __percpu *stats = NULL;
2459                 struct nft_chain_hook hook = {};
2460
2461                 if (table->flags & __NFT_TABLE_F_UPDATE)
2462                         return -EINVAL;
2463
2464                 if (flags & NFT_CHAIN_BINDING)
2465                         return -EOPNOTSUPP;
2466
2467                 err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
2468                                            extack);
2469                 if (err < 0)
2470                         return err;
2471
2472                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
2473                 if (basechain == NULL) {
2474                         nft_chain_release_hook(&hook);
2475                         return -ENOMEM;
2476                 }
2477                 chain = &basechain->chain;
2478
2479                 if (nla[NFTA_CHAIN_COUNTERS]) {
2480                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2481                         if (IS_ERR(stats)) {
2482                                 nft_chain_release_hook(&hook);
2483                                 kfree(basechain);
2484                                 return PTR_ERR(stats);
2485                         }
2486                         rcu_assign_pointer(basechain->stats, stats);
2487                 }
2488
2489                 err = nft_basechain_init(basechain, family, &hook, flags);
2490                 if (err < 0) {
2491                         nft_chain_release_hook(&hook);
2492                         kfree(basechain);
2493                         free_percpu(stats);
2494                         return err;
2495                 }
2496                 if (stats)
2497                         static_branch_inc(&nft_counters_enabled);
2498         } else {
2499                 if (flags & NFT_CHAIN_BASE)
2500                         return -EINVAL;
2501                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2502                         return -EOPNOTSUPP;
2503
2504                 chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
2505                 if (chain == NULL)
2506                         return -ENOMEM;
2507
2508                 chain->flags = flags;
2509         }
2510         ctx->chain = chain;
2511
2512         INIT_LIST_HEAD(&chain->rules);
2513         chain->handle = nf_tables_alloc_handle(table);
2514         chain->table = table;
2515
2516         if (nla[NFTA_CHAIN_NAME]) {
2517                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2518         } else {
2519                 if (!(flags & NFT_CHAIN_BINDING)) {
2520                         err = -EINVAL;
2521                         goto err_destroy_chain;
2522                 }
2523
2524                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2525                 chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
2526         }
2527
2528         if (!chain->name) {
2529                 err = -ENOMEM;
2530                 goto err_destroy_chain;
2531         }
2532
2533         if (nla[NFTA_CHAIN_USERDATA]) {
2534                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
2535                 if (chain->udata == NULL) {
2536                         err = -ENOMEM;
2537                         goto err_destroy_chain;
2538                 }
2539                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2540         }
2541
2542         blob = nf_tables_chain_alloc_rules(chain, 0);
2543         if (!blob) {
2544                 err = -ENOMEM;
2545                 goto err_destroy_chain;
2546         }
2547
2548         RCU_INIT_POINTER(chain->blob_gen_0, blob);
2549         RCU_INIT_POINTER(chain->blob_gen_1, blob);
2550
2551         if (!nft_use_inc(&table->use)) {
2552                 err = -EMFILE;
2553                 goto err_destroy_chain;
2554         }
2555
2556         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2557         if (IS_ERR(trans)) {
2558                 err = PTR_ERR(trans);
2559                 goto err_trans;
2560         }
2561
2562         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2563         if (nft_is_base_chain(chain))
2564                 nft_trans_chain_policy(trans) = policy;
2565
2566         err = nft_chain_add(table, chain);
2567         if (err < 0)
2568                 goto err_chain_add;
2569
2570         /* This must be LAST to ensure no packets are walking over this chain. */
2571         err = nf_tables_register_hook(net, table, chain);
2572         if (err < 0)
2573                 goto err_register_hook;
2574
2575         return 0;
2576
2577 err_register_hook:
2578         nft_chain_del(chain);
2579 err_chain_add:
2580         nft_trans_destroy(trans);
2581 err_trans:
2582         nft_use_dec_restore(&table->use);
2583 err_destroy_chain:
2584         nf_tables_chain_destroy(ctx);
2585
2586         return err;
2587 }
2588
2589 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2590                               u32 flags, const struct nlattr *attr,
2591                               struct netlink_ext_ack *extack)
2592 {
2593         const struct nlattr * const *nla = ctx->nla;
2594         struct nft_base_chain *basechain = NULL;
2595         struct nft_table *table = ctx->table;
2596         struct nft_chain *chain = ctx->chain;
2597         struct nft_chain_hook hook = {};
2598         struct nft_stats *stats = NULL;
2599         struct nft_hook *h, *next;
2600         struct nf_hook_ops *ops;
2601         struct nft_trans *trans;
2602         bool unregister = false;
2603         int err;
2604
2605         if (chain->flags ^ flags)
2606                 return -EOPNOTSUPP;
2607
2608         INIT_LIST_HEAD(&hook.list);
2609
2610         if (nla[NFTA_CHAIN_HOOK]) {
2611                 if (!nft_is_base_chain(chain)) {
2612                         NL_SET_BAD_ATTR(extack, attr);
2613                         return -EEXIST;
2614                 }
2615
2616                 basechain = nft_base_chain(chain);
2617                 err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
2618                                            ctx->family, flags, extack);
2619                 if (err < 0)
2620                         return err;
2621
2622                 if (basechain->type != hook.type) {
2623                         nft_chain_release_hook(&hook);
2624                         NL_SET_BAD_ATTR(extack, attr);
2625                         return -EEXIST;
2626                 }
2627
2628                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
2629                         list_for_each_entry_safe(h, next, &hook.list, list) {
2630                                 h->ops.pf       = basechain->ops.pf;
2631                                 h->ops.hooknum  = basechain->ops.hooknum;
2632                                 h->ops.priority = basechain->ops.priority;
2633                                 h->ops.priv     = basechain->ops.priv;
2634                                 h->ops.hook     = basechain->ops.hook;
2635
2636                                 if (nft_hook_list_find(&basechain->hook_list, h)) {
2637                                         list_del(&h->list);
2638                                         kfree(h);
2639                                 }
2640                         }
2641                 } else {
2642                         ops = &basechain->ops;
2643                         if (ops->hooknum != hook.num ||
2644                             ops->priority != hook.priority) {
2645                                 nft_chain_release_hook(&hook);
2646                                 NL_SET_BAD_ATTR(extack, attr);
2647                                 return -EEXIST;
2648                         }
2649                 }
2650         }
2651
2652         if (nla[NFTA_CHAIN_HANDLE] &&
2653             nla[NFTA_CHAIN_NAME]) {
2654                 struct nft_chain *chain2;
2655
2656                 chain2 = nft_chain_lookup(ctx->net, table,
2657                                           nla[NFTA_CHAIN_NAME], genmask);
2658                 if (!IS_ERR(chain2)) {
2659                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2660                         err = -EEXIST;
2661                         goto err_hooks;
2662                 }
2663         }
2664
2665         if (table->flags & __NFT_TABLE_F_UPDATE &&
2666             !list_empty(&hook.list)) {
2667                 NL_SET_BAD_ATTR(extack, attr);
2668                 err = -EOPNOTSUPP;
2669                 goto err_hooks;
2670         }
2671
2672         if (!(table->flags & NFT_TABLE_F_DORMANT) &&
2673             nft_is_base_chain(chain) &&
2674             !list_empty(&hook.list)) {
2675                 basechain = nft_base_chain(chain);
2676                 ops = &basechain->ops;
2677
2678                 if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
2679                         err = nft_netdev_register_hooks(ctx->net, &hook.list);
2680                         if (err < 0)
2681                                 goto err_hooks;
2682                 }
2683         }
2684
2685         unregister = true;
2686
2687         if (nla[NFTA_CHAIN_COUNTERS]) {
2688                 if (!nft_is_base_chain(chain)) {
2689                         err = -EOPNOTSUPP;
2690                         goto err_hooks;
2691                 }
2692
2693                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2694                 if (IS_ERR(stats)) {
2695                         err = PTR_ERR(stats);
2696                         goto err_hooks;
2697                 }
2698         }
2699
2700         err = -ENOMEM;
2701         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2702                                 sizeof(struct nft_trans_chain));
2703         if (trans == NULL)
2704                 goto err_trans;
2705
2706         nft_trans_chain_stats(trans) = stats;
2707         nft_trans_chain_update(trans) = true;
2708
2709         if (nla[NFTA_CHAIN_POLICY])
2710                 nft_trans_chain_policy(trans) = policy;
2711         else
2712                 nft_trans_chain_policy(trans) = -1;
2713
2714         if (nla[NFTA_CHAIN_HANDLE] &&
2715             nla[NFTA_CHAIN_NAME]) {
2716                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2717                 struct nft_trans *tmp;
2718                 char *name;
2719
2720                 err = -ENOMEM;
2721                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
2722                 if (!name)
2723                         goto err_trans;
2724
2725                 err = -EEXIST;
2726                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2727                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2728                             tmp->ctx.table == table &&
2729                             nft_trans_chain_update(tmp) &&
2730                             nft_trans_chain_name(tmp) &&
2731                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2732                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2733                                 kfree(name);
2734                                 goto err_trans;
2735                         }
2736                 }
2737
2738                 nft_trans_chain_name(trans) = name;
2739         }
2740
2741         nft_trans_basechain(trans) = basechain;
2742         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2743         list_splice(&hook.list, &nft_trans_chain_hooks(trans));
2744         if (nla[NFTA_CHAIN_HOOK])
2745                 module_put(hook.type->owner);
2746
2747         nft_trans_commit_list_add_tail(ctx->net, trans);
2748
2749         return 0;
2750
2751 err_trans:
2752         free_percpu(stats);
2753         kfree(trans);
2754 err_hooks:
2755         if (nla[NFTA_CHAIN_HOOK]) {
2756                 list_for_each_entry_safe(h, next, &hook.list, list) {
2757                         if (unregister)
2758                                 nf_unregister_net_hook(ctx->net, &h->ops);
2759                         list_del(&h->list);
2760                         kfree_rcu(h, rcu);
2761                 }
2762                 module_put(hook.type->owner);
2763         }
2764
2765         return err;
2766 }
2767
2768 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2769                                                const struct nft_table *table,
2770                                                const struct nlattr *nla, u8 genmask)
2771 {
2772         struct nftables_pernet *nft_net = nft_pernet(net);
2773         u32 id = ntohl(nla_get_be32(nla));
2774         struct nft_trans *trans;
2775
2776         list_for_each_entry(trans, &nft_net->commit_list, list) {
2777                 struct nft_chain *chain = trans->ctx.chain;
2778
2779                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2780                     chain->table == table &&
2781                     id == nft_trans_chain_id(trans) &&
2782                     nft_active_genmask(chain, genmask))
2783                         return chain;
2784         }
2785         return ERR_PTR(-ENOENT);
2786 }
2787
2788 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2789                               const struct nlattr * const nla[])
2790 {
2791         struct nftables_pernet *nft_net = nft_pernet(info->net);
2792         struct netlink_ext_ack *extack = info->extack;
2793         u8 genmask = nft_genmask_next(info->net);
2794         u8 family = info->nfmsg->nfgen_family;
2795         struct nft_chain *chain = NULL;
2796         struct net *net = info->net;
2797         const struct nlattr *attr;
2798         struct nft_table *table;
2799         u8 policy = NF_ACCEPT;
2800         struct nft_ctx ctx;
2801         u64 handle = 0;
2802         u32 flags = 0;
2803
2804         lockdep_assert_held(&nft_net->commit_mutex);
2805
2806         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2807                                  NETLINK_CB(skb).portid);
2808         if (IS_ERR(table)) {
2809                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2810                 return PTR_ERR(table);
2811         }
2812
2813         chain = NULL;
2814         attr = nla[NFTA_CHAIN_NAME];
2815
2816         if (nla[NFTA_CHAIN_HANDLE]) {
2817                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2818                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2819                 if (IS_ERR(chain)) {
2820                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2821                         return PTR_ERR(chain);
2822                 }
2823                 attr = nla[NFTA_CHAIN_HANDLE];
2824         } else if (nla[NFTA_CHAIN_NAME]) {
2825                 chain = nft_chain_lookup(net, table, attr, genmask);
2826                 if (IS_ERR(chain)) {
2827                         if (PTR_ERR(chain) != -ENOENT) {
2828                                 NL_SET_BAD_ATTR(extack, attr);
2829                                 return PTR_ERR(chain);
2830                         }
2831                         chain = NULL;
2832                 }
2833         } else if (!nla[NFTA_CHAIN_ID]) {
2834                 return -EINVAL;
2835         }
2836
2837         if (nla[NFTA_CHAIN_POLICY]) {
2838                 if (chain != NULL &&
2839                     !nft_is_base_chain(chain)) {
2840                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2841                         return -EOPNOTSUPP;
2842                 }
2843
2844                 if (chain == NULL &&
2845                     nla[NFTA_CHAIN_HOOK] == NULL) {
2846                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2847                         return -EOPNOTSUPP;
2848                 }
2849
2850                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2851                 switch (policy) {
2852                 case NF_DROP:
2853                 case NF_ACCEPT:
2854                         break;
2855                 default:
2856                         return -EINVAL;
2857                 }
2858         }
2859
2860         if (nla[NFTA_CHAIN_FLAGS])
2861                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2862         else if (chain)
2863                 flags = chain->flags;
2864
2865         if (flags & ~NFT_CHAIN_FLAGS)
2866                 return -EOPNOTSUPP;
2867
2868         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2869
2870         if (chain != NULL) {
2871                 if (chain->flags & NFT_CHAIN_BINDING)
2872                         return -EINVAL;
2873
2874                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2875                         NL_SET_BAD_ATTR(extack, attr);
2876                         return -EEXIST;
2877                 }
2878                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2879                         return -EOPNOTSUPP;
2880
2881                 flags |= chain->flags & NFT_CHAIN_BASE;
2882                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2883                                           extack);
2884         }
2885
2886         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2887 }
2888
2889 static int nft_delchain_hook(struct nft_ctx *ctx,
2890                              struct nft_base_chain *basechain,
2891                              struct netlink_ext_ack *extack)
2892 {
2893         const struct nft_chain *chain = &basechain->chain;
2894         const struct nlattr * const *nla = ctx->nla;
2895         struct nft_chain_hook chain_hook = {};
2896         struct nft_hook *this, *hook;
2897         LIST_HEAD(chain_del_list);
2898         struct nft_trans *trans;
2899         int err;
2900
2901         if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
2902                 return -EOPNOTSUPP;
2903
2904         err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
2905                                    ctx->family, chain->flags, extack);
2906         if (err < 0)
2907                 return err;
2908
2909         list_for_each_entry(this, &chain_hook.list, list) {
2910                 hook = nft_hook_list_find(&basechain->hook_list, this);
2911                 if (!hook) {
2912                         err = -ENOENT;
2913                         goto err_chain_del_hook;
2914                 }
2915                 list_move(&hook->list, &chain_del_list);
2916         }
2917
2918         trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN,
2919                                 sizeof(struct nft_trans_chain));
2920         if (!trans) {
2921                 err = -ENOMEM;
2922                 goto err_chain_del_hook;
2923         }
2924
2925         nft_trans_basechain(trans) = basechain;
2926         nft_trans_chain_update(trans) = true;
2927         INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
2928         list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
2929         nft_chain_release_hook(&chain_hook);
2930
2931         nft_trans_commit_list_add_tail(ctx->net, trans);
2932
2933         return 0;
2934
2935 err_chain_del_hook:
2936         list_splice(&chain_del_list, &basechain->hook_list);
2937         nft_chain_release_hook(&chain_hook);
2938
2939         return err;
2940 }
2941
2942 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2943                               const struct nlattr * const nla[])
2944 {
2945         struct netlink_ext_ack *extack = info->extack;
2946         u8 genmask = nft_genmask_next(info->net);
2947         u8 family = info->nfmsg->nfgen_family;
2948         struct net *net = info->net;
2949         const struct nlattr *attr;
2950         struct nft_table *table;
2951         struct nft_chain *chain;
2952         struct nft_rule *rule;
2953         struct nft_ctx ctx;
2954         u64 handle;
2955         u32 use;
2956         int err;
2957
2958         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2959                                  NETLINK_CB(skb).portid);
2960         if (IS_ERR(table)) {
2961                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2962                 return PTR_ERR(table);
2963         }
2964
2965         if (nla[NFTA_CHAIN_HANDLE]) {
2966                 attr = nla[NFTA_CHAIN_HANDLE];
2967                 handle = be64_to_cpu(nla_get_be64(attr));
2968                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2969         } else {
2970                 attr = nla[NFTA_CHAIN_NAME];
2971                 chain = nft_chain_lookup(net, table, attr, genmask);
2972         }
2973         if (IS_ERR(chain)) {
2974                 if (PTR_ERR(chain) == -ENOENT &&
2975                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
2976                         return 0;
2977
2978                 NL_SET_BAD_ATTR(extack, attr);
2979                 return PTR_ERR(chain);
2980         }
2981
2982         if (nft_chain_binding(chain))
2983                 return -EOPNOTSUPP;
2984
2985         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2986
2987         if (nla[NFTA_CHAIN_HOOK]) {
2988                 if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN ||
2989                     chain->flags & NFT_CHAIN_HW_OFFLOAD)
2990                         return -EOPNOTSUPP;
2991
2992                 if (nft_is_base_chain(chain)) {
2993                         struct nft_base_chain *basechain = nft_base_chain(chain);
2994
2995                         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
2996                                 return nft_delchain_hook(&ctx, basechain, extack);
2997                 }
2998         }
2999
3000         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
3001             chain->use > 0)
3002                 return -EBUSY;
3003
3004         use = chain->use;
3005         list_for_each_entry(rule, &chain->rules, list) {
3006                 if (!nft_is_active_next(net, rule))
3007                         continue;
3008                 use--;
3009
3010                 err = nft_delrule(&ctx, rule);
3011                 if (err < 0)
3012                         return err;
3013         }
3014
3015         /* There are rules and elements that are still holding references to us,
3016          * we cannot do a recursive removal in this case.
3017          */
3018         if (use > 0) {
3019                 NL_SET_BAD_ATTR(extack, attr);
3020                 return -EBUSY;
3021         }
3022
3023         return nft_delchain(&ctx);
3024 }
3025
3026 /*
3027  * Expressions
3028  */
3029
3030 /**
3031  *      nft_register_expr - register nf_tables expr type
3032  *      @type: expr type
3033  *
3034  *      Registers the expr type for use with nf_tables. Returns zero on
3035  *      success or a negative errno code otherwise.
3036  */
3037 int nft_register_expr(struct nft_expr_type *type)
3038 {
3039         if (WARN_ON_ONCE(type->maxattr > NFT_EXPR_MAXATTR))
3040                 return -ENOMEM;
3041
3042         nfnl_lock(NFNL_SUBSYS_NFTABLES);
3043         if (type->family == NFPROTO_UNSPEC)
3044                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
3045         else
3046                 list_add_rcu(&type->list, &nf_tables_expressions);
3047         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
3048         return 0;
3049 }
3050 EXPORT_SYMBOL_GPL(nft_register_expr);
3051
3052 /**
3053  *      nft_unregister_expr - unregister nf_tables expr type
3054  *      @type: expr type
3055  *
3056  *      Unregisters the expr typefor use with nf_tables.
3057  */
3058 void nft_unregister_expr(struct nft_expr_type *type)
3059 {
3060         nfnl_lock(NFNL_SUBSYS_NFTABLES);
3061         list_del_rcu(&type->list);
3062         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
3063 }
3064 EXPORT_SYMBOL_GPL(nft_unregister_expr);
3065
3066 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
3067                                                        struct nlattr *nla)
3068 {
3069         const struct nft_expr_type *type, *candidate = NULL;
3070
3071         list_for_each_entry_rcu(type, &nf_tables_expressions, list) {
3072                 if (!nla_strcmp(nla, type->name)) {
3073                         if (!type->family && !candidate)
3074                                 candidate = type;
3075                         else if (type->family == family)
3076                                 candidate = type;
3077                 }
3078         }
3079         return candidate;
3080 }
3081
3082 #ifdef CONFIG_MODULES
3083 static int nft_expr_type_request_module(struct net *net, u8 family,
3084                                         struct nlattr *nla)
3085 {
3086         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
3087                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
3088                 return -EAGAIN;
3089
3090         return 0;
3091 }
3092 #endif
3093
3094 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
3095                                                      u8 family,
3096                                                      struct nlattr *nla)
3097 {
3098         const struct nft_expr_type *type;
3099
3100         if (nla == NULL)
3101                 return ERR_PTR(-EINVAL);
3102
3103         rcu_read_lock();
3104         type = __nft_expr_type_get(family, nla);
3105         if (type != NULL && try_module_get(type->owner)) {
3106                 rcu_read_unlock();
3107                 return type;
3108         }
3109         rcu_read_unlock();
3110
3111         lockdep_nfnl_nft_mutex_not_held();
3112 #ifdef CONFIG_MODULES
3113         if (type == NULL) {
3114                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
3115                         return ERR_PTR(-EAGAIN);
3116
3117                 if (nft_request_module(net, "nft-expr-%.*s",
3118                                        nla_len(nla),
3119                                        (char *)nla_data(nla)) == -EAGAIN)
3120                         return ERR_PTR(-EAGAIN);
3121         }
3122 #endif
3123         return ERR_PTR(-ENOENT);
3124 }
3125
3126 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
3127         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
3128                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
3129         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
3130 };
3131
3132 static int nf_tables_fill_expr_info(struct sk_buff *skb,
3133                                     const struct nft_expr *expr, bool reset)
3134 {
3135         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
3136                 goto nla_put_failure;
3137
3138         if (expr->ops->dump) {
3139                 struct nlattr *data = nla_nest_start_noflag(skb,
3140                                                             NFTA_EXPR_DATA);
3141                 if (data == NULL)
3142                         goto nla_put_failure;
3143                 if (expr->ops->dump(skb, expr, reset) < 0)
3144                         goto nla_put_failure;
3145                 nla_nest_end(skb, data);
3146         }
3147
3148         return skb->len;
3149
3150 nla_put_failure:
3151         return -1;
3152 };
3153
3154 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
3155                   const struct nft_expr *expr, bool reset)
3156 {
3157         struct nlattr *nest;
3158
3159         nest = nla_nest_start_noflag(skb, attr);
3160         if (!nest)
3161                 goto nla_put_failure;
3162         if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
3163                 goto nla_put_failure;
3164         nla_nest_end(skb, nest);
3165         return 0;
3166
3167 nla_put_failure:
3168         return -1;
3169 }
3170
3171 struct nft_expr_info {
3172         const struct nft_expr_ops       *ops;
3173         const struct nlattr             *attr;
3174         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
3175 };
3176
3177 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
3178                                 const struct nlattr *nla,
3179                                 struct nft_expr_info *info)
3180 {
3181         const struct nft_expr_type *type;
3182         const struct nft_expr_ops *ops;
3183         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3184         int err;
3185
3186         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3187                                           nft_expr_policy, NULL);
3188         if (err < 0)
3189                 return err;
3190
3191         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
3192         if (IS_ERR(type))
3193                 return PTR_ERR(type);
3194
3195         if (tb[NFTA_EXPR_DATA]) {
3196                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3197                                                   tb[NFTA_EXPR_DATA],
3198                                                   type->policy, NULL);
3199                 if (err < 0)
3200                         goto err1;
3201         } else
3202                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
3203
3204         if (type->select_ops != NULL) {
3205                 ops = type->select_ops(ctx,
3206                                        (const struct nlattr * const *)info->tb);
3207                 if (IS_ERR(ops)) {
3208                         err = PTR_ERR(ops);
3209 #ifdef CONFIG_MODULES
3210                         if (err == -EAGAIN)
3211                                 if (nft_expr_type_request_module(ctx->net,
3212                                                                  ctx->family,
3213                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
3214                                         err = -ENOENT;
3215 #endif
3216                         goto err1;
3217                 }
3218         } else
3219                 ops = type->ops;
3220
3221         info->attr = nla;
3222         info->ops = ops;
3223
3224         return 0;
3225
3226 err1:
3227         module_put(type->owner);
3228         return err;
3229 }
3230
3231 int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
3232                          struct nft_expr_info *info)
3233 {
3234         struct nlattr *tb[NFTA_EXPR_MAX + 1];
3235         const struct nft_expr_type *type;
3236         int err;
3237
3238         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
3239                                           nft_expr_policy, NULL);
3240         if (err < 0)
3241                 return err;
3242
3243         if (!tb[NFTA_EXPR_DATA] || !tb[NFTA_EXPR_NAME])
3244                 return -EINVAL;
3245
3246         type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
3247         if (!type)
3248                 return -ENOENT;
3249
3250         if (!type->inner_ops)
3251                 return -EOPNOTSUPP;
3252
3253         err = nla_parse_nested_deprecated(info->tb, type->maxattr,
3254                                           tb[NFTA_EXPR_DATA],
3255                                           type->policy, NULL);
3256         if (err < 0)
3257                 goto err_nla_parse;
3258
3259         info->attr = nla;
3260         info->ops = type->inner_ops;
3261
3262         return 0;
3263
3264 err_nla_parse:
3265         return err;
3266 }
3267
3268 static int nf_tables_newexpr(const struct nft_ctx *ctx,
3269                              const struct nft_expr_info *expr_info,
3270                              struct nft_expr *expr)
3271 {
3272         const struct nft_expr_ops *ops = expr_info->ops;
3273         int err;
3274
3275         expr->ops = ops;
3276         if (ops->init) {
3277                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
3278                 if (err < 0)
3279                         goto err1;
3280         }
3281
3282         return 0;
3283 err1:
3284         expr->ops = NULL;
3285         return err;
3286 }
3287
3288 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
3289                                    struct nft_expr *expr)
3290 {
3291         const struct nft_expr_type *type = expr->ops->type;
3292
3293         if (expr->ops->destroy)
3294                 expr->ops->destroy(ctx, expr);
3295         module_put(type->owner);
3296 }
3297
3298 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
3299                                       const struct nlattr *nla)
3300 {
3301         struct nft_expr_info expr_info;
3302         struct nft_expr *expr;
3303         struct module *owner;
3304         int err;
3305
3306         err = nf_tables_expr_parse(ctx, nla, &expr_info);
3307         if (err < 0)
3308                 goto err_expr_parse;
3309
3310         err = -EOPNOTSUPP;
3311         if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
3312                 goto err_expr_stateful;
3313
3314         err = -ENOMEM;
3315         expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
3316         if (expr == NULL)
3317                 goto err_expr_stateful;
3318
3319         err = nf_tables_newexpr(ctx, &expr_info, expr);
3320         if (err < 0)
3321                 goto err_expr_new;
3322
3323         return expr;
3324 err_expr_new:
3325         kfree(expr);
3326 err_expr_stateful:
3327         owner = expr_info.ops->type->owner;
3328         if (expr_info.ops->type->release_ops)
3329                 expr_info.ops->type->release_ops(expr_info.ops);
3330
3331         module_put(owner);
3332 err_expr_parse:
3333         return ERR_PTR(err);
3334 }
3335
3336 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
3337 {
3338         int err;
3339
3340         if (WARN_ON_ONCE(!src->ops->clone))
3341                 return -EINVAL;
3342
3343         dst->ops = src->ops;
3344         err = src->ops->clone(dst, src);
3345         if (err < 0)
3346                 return err;
3347
3348         __module_get(src->ops->type->owner);
3349
3350         return 0;
3351 }
3352
3353 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
3354 {
3355         nf_tables_expr_destroy(ctx, expr);
3356         kfree(expr);
3357 }
3358
3359 /*
3360  * Rules
3361  */
3362
3363 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
3364                                           u64 handle)
3365 {
3366         struct nft_rule *rule;
3367
3368         // FIXME: this sucks
3369         list_for_each_entry_rcu(rule, &chain->rules, list) {
3370                 if (handle == rule->handle)
3371                         return rule;
3372         }
3373
3374         return ERR_PTR(-ENOENT);
3375 }
3376
3377 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
3378                                         const struct nlattr *nla)
3379 {
3380         if (nla == NULL)
3381                 return ERR_PTR(-EINVAL);
3382
3383         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
3384 }
3385
3386 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
3387         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
3388                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
3389         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
3390                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
3391         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
3392         [NFTA_RULE_EXPRESSIONS] = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
3393         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
3394         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
3395         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
3396                                     .len = NFT_USERDATA_MAXLEN },
3397         [NFTA_RULE_ID]          = { .type = NLA_U32 },
3398         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
3399         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
3400 };
3401
3402 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
3403                                     u32 portid, u32 seq, int event,
3404                                     u32 flags, int family,
3405                                     const struct nft_table *table,
3406                                     const struct nft_chain *chain,
3407                                     const struct nft_rule *rule, u64 handle,
3408                                     bool reset)
3409 {
3410         struct nlmsghdr *nlh;
3411         const struct nft_expr *expr, *next;
3412         struct nlattr *list;
3413         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3414
3415         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
3416                            nft_base_seq(net));
3417         if (!nlh)
3418                 goto nla_put_failure;
3419
3420         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
3421                 goto nla_put_failure;
3422         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
3423                 goto nla_put_failure;
3424         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
3425                          NFTA_RULE_PAD))
3426                 goto nla_put_failure;
3427
3428         if (event != NFT_MSG_DELRULE && handle) {
3429                 if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
3430                                  NFTA_RULE_PAD))
3431                         goto nla_put_failure;
3432         }
3433
3434         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
3435                 nft_flow_rule_stats(chain, rule);
3436
3437         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
3438         if (list == NULL)
3439                 goto nla_put_failure;
3440         nft_rule_for_each_expr(expr, next, rule) {
3441                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
3442                         goto nla_put_failure;
3443         }
3444         nla_nest_end(skb, list);
3445
3446         if (rule->udata) {
3447                 struct nft_userdata *udata = nft_userdata(rule);
3448                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
3449                             udata->data) < 0)
3450                         goto nla_put_failure;
3451         }
3452
3453         nlmsg_end(skb, nlh);
3454         return 0;
3455
3456 nla_put_failure:
3457         nlmsg_trim(skb, nlh);
3458         return -1;
3459 }
3460
3461 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
3462                                   const struct nft_rule *rule, int event)
3463 {
3464         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3465         const struct nft_rule *prule;
3466         struct sk_buff *skb;
3467         u64 handle = 0;
3468         u16 flags = 0;
3469         int err;
3470
3471         if (!ctx->report &&
3472             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3473                 return;
3474
3475         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
3476         if (skb == NULL)
3477                 goto err;
3478
3479         if (event == NFT_MSG_NEWRULE &&
3480             !list_is_first(&rule->list, &ctx->chain->rules) &&
3481             !list_is_last(&rule->list, &ctx->chain->rules)) {
3482                 prule = list_prev_entry(rule, list);
3483                 handle = prule->handle;
3484         }
3485         if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
3486                 flags |= NLM_F_APPEND;
3487         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
3488                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
3489
3490         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
3491                                        event, flags, ctx->family, ctx->table,
3492                                        ctx->chain, rule, handle, false);
3493         if (err < 0) {
3494                 kfree_skb(skb);
3495                 goto err;
3496         }
3497
3498         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3499         return;
3500 err:
3501         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
3502 }
3503
3504 static void audit_log_rule_reset(const struct nft_table *table,
3505                                  unsigned int base_seq,
3506                                  unsigned int nentries)
3507 {
3508         char *buf = kasprintf(GFP_ATOMIC, "%s:%u",
3509                               table->name, base_seq);
3510
3511         audit_log_nfcfg(buf, table->family, nentries,
3512                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3513         kfree(buf);
3514 }
3515
3516 struct nft_rule_dump_ctx {
3517         unsigned int s_idx;
3518         char *table;
3519         char *chain;
3520         bool reset;
3521 };
3522
3523 static int __nf_tables_dump_rules(struct sk_buff *skb,
3524                                   unsigned int *idx,
3525                                   struct netlink_callback *cb,
3526                                   const struct nft_table *table,
3527                                   const struct nft_chain *chain)
3528 {
3529         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3530         struct net *net = sock_net(skb->sk);
3531         const struct nft_rule *rule, *prule;
3532         unsigned int entries = 0;
3533         int ret = 0;
3534         u64 handle;
3535
3536         prule = NULL;
3537         list_for_each_entry_rcu(rule, &chain->rules, list) {
3538                 if (!nft_is_active(net, rule))
3539                         goto cont_skip;
3540                 if (*idx < ctx->s_idx)
3541                         goto cont;
3542                 if (prule)
3543                         handle = prule->handle;
3544                 else
3545                         handle = 0;
3546
3547                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
3548                                         cb->nlh->nlmsg_seq,
3549                                         NFT_MSG_NEWRULE,
3550                                         NLM_F_MULTI | NLM_F_APPEND,
3551                                         table->family,
3552                                         table, chain, rule, handle, ctx->reset) < 0) {
3553                         ret = 1;
3554                         break;
3555                 }
3556                 entries++;
3557                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3558 cont:
3559                 prule = rule;
3560 cont_skip:
3561                 (*idx)++;
3562         }
3563
3564         if (ctx->reset && entries)
3565                 audit_log_rule_reset(table, cb->seq, entries);
3566
3567         return ret;
3568 }
3569
3570 static int nf_tables_dump_rules(struct sk_buff *skb,
3571                                 struct netlink_callback *cb)
3572 {
3573         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
3574         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3575         struct nft_table *table;
3576         const struct nft_chain *chain;
3577         unsigned int idx = 0;
3578         struct net *net = sock_net(skb->sk);
3579         int family = nfmsg->nfgen_family;
3580         struct nftables_pernet *nft_net;
3581
3582         rcu_read_lock();
3583         nft_net = nft_pernet(net);
3584         cb->seq = READ_ONCE(nft_net->base_seq);
3585
3586         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3587                 if (family != NFPROTO_UNSPEC && family != table->family)
3588                         continue;
3589
3590                 if (ctx->table && strcmp(ctx->table, table->name) != 0)
3591                         continue;
3592
3593                 if (ctx->table && ctx->chain) {
3594                         struct rhlist_head *list, *tmp;
3595
3596                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3597                                                nft_chain_ht_params);
3598                         if (!list)
3599                                 goto done;
3600
3601                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3602                                 if (!nft_is_active(net, chain))
3603                                         continue;
3604                                 __nf_tables_dump_rules(skb, &idx,
3605                                                        cb, table, chain);
3606                                 break;
3607                         }
3608                         goto done;
3609                 }
3610
3611                 list_for_each_entry_rcu(chain, &table->chains, list) {
3612                         if (__nf_tables_dump_rules(skb, &idx,
3613                                                    cb, table, chain))
3614                                 goto done;
3615                 }
3616
3617                 if (ctx->table)
3618                         break;
3619         }
3620 done:
3621         rcu_read_unlock();
3622
3623         ctx->s_idx = idx;
3624         return skb->len;
3625 }
3626
3627 static int nf_tables_dumpreset_rules(struct sk_buff *skb,
3628                                      struct netlink_callback *cb)
3629 {
3630         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
3631         int ret;
3632
3633         /* Mutex is held is to prevent that two concurrent dump-and-reset calls
3634          * do not underrun counters and quotas. The commit_mutex is used for
3635          * the lack a better lock, this is not transaction path.
3636          */
3637         mutex_lock(&nft_net->commit_mutex);
3638         ret = nf_tables_dump_rules(skb, cb);
3639         mutex_unlock(&nft_net->commit_mutex);
3640
3641         return ret;
3642 }
3643
3644 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3645 {
3646         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3647         const struct nlattr * const *nla = cb->data;
3648
3649         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
3650
3651         if (nla[NFTA_RULE_TABLE]) {
3652                 ctx->table = nla_strdup(nla[NFTA_RULE_TABLE], GFP_ATOMIC);
3653                 if (!ctx->table)
3654                         return -ENOMEM;
3655         }
3656         if (nla[NFTA_RULE_CHAIN]) {
3657                 ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN], GFP_ATOMIC);
3658                 if (!ctx->chain) {
3659                         kfree(ctx->table);
3660                         return -ENOMEM;
3661                 }
3662         }
3663         return 0;
3664 }
3665
3666 static int nf_tables_dumpreset_rules_start(struct netlink_callback *cb)
3667 {
3668         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3669
3670         ctx->reset = true;
3671
3672         return nf_tables_dump_rules_start(cb);
3673 }
3674
3675 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3676 {
3677         struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
3678
3679         kfree(ctx->table);
3680         kfree(ctx->chain);
3681         return 0;
3682 }
3683
3684 /* called with rcu_read_lock held */
3685 static struct sk_buff *
3686 nf_tables_getrule_single(u32 portid, const struct nfnl_info *info,
3687                          const struct nlattr * const nla[], bool reset)
3688 {
3689         struct netlink_ext_ack *extack = info->extack;
3690         u8 genmask = nft_genmask_cur(info->net);
3691         u8 family = info->nfmsg->nfgen_family;
3692         const struct nft_chain *chain;
3693         const struct nft_rule *rule;
3694         struct net *net = info->net;
3695         struct nft_table *table;
3696         struct sk_buff *skb2;
3697         int err;
3698
3699         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3700         if (IS_ERR(table)) {
3701                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3702                 return ERR_CAST(table);
3703         }
3704
3705         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3706         if (IS_ERR(chain)) {
3707                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3708                 return ERR_CAST(chain);
3709         }
3710
3711         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3712         if (IS_ERR(rule)) {
3713                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3714                 return ERR_CAST(rule);
3715         }
3716
3717         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3718         if (!skb2)
3719                 return ERR_PTR(-ENOMEM);
3720
3721         err = nf_tables_fill_rule_info(skb2, net, portid,
3722                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3723                                        family, table, chain, rule, 0, reset);
3724         if (err < 0) {
3725                 kfree_skb(skb2);
3726                 return ERR_PTR(err);
3727         }
3728
3729         return skb2;
3730 }
3731
3732 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3733                              const struct nlattr * const nla[])
3734 {
3735         u32 portid = NETLINK_CB(skb).portid;
3736         struct net *net = info->net;
3737         struct sk_buff *skb2;
3738
3739         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3740                 struct netlink_dump_control c = {
3741                         .start= nf_tables_dump_rules_start,
3742                         .dump = nf_tables_dump_rules,
3743                         .done = nf_tables_dump_rules_done,
3744                         .module = THIS_MODULE,
3745                         .data = (void *)nla,
3746                 };
3747
3748                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3749         }
3750
3751         skb2 = nf_tables_getrule_single(portid, info, nla, false);
3752         if (IS_ERR(skb2))
3753                 return PTR_ERR(skb2);
3754
3755         return nfnetlink_unicast(skb2, net, portid);
3756 }
3757
3758 static int nf_tables_getrule_reset(struct sk_buff *skb,
3759                                    const struct nfnl_info *info,
3760                                    const struct nlattr * const nla[])
3761 {
3762         struct nftables_pernet *nft_net = nft_pernet(info->net);
3763         u32 portid = NETLINK_CB(skb).portid;
3764         struct net *net = info->net;
3765         struct sk_buff *skb2;
3766         char *buf;
3767
3768         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3769                 struct netlink_dump_control c = {
3770                         .start= nf_tables_dumpreset_rules_start,
3771                         .dump = nf_tables_dumpreset_rules,
3772                         .done = nf_tables_dump_rules_done,
3773                         .module = THIS_MODULE,
3774                         .data = (void *)nla,
3775                 };
3776
3777                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3778         }
3779
3780         if (!try_module_get(THIS_MODULE))
3781                 return -EINVAL;
3782         rcu_read_unlock();
3783         mutex_lock(&nft_net->commit_mutex);
3784         skb2 = nf_tables_getrule_single(portid, info, nla, true);
3785         mutex_unlock(&nft_net->commit_mutex);
3786         rcu_read_lock();
3787         module_put(THIS_MODULE);
3788
3789         if (IS_ERR(skb2))
3790                 return PTR_ERR(skb2);
3791
3792         buf = kasprintf(GFP_ATOMIC, "%.*s:%u",
3793                         nla_len(nla[NFTA_RULE_TABLE]),
3794                         (char *)nla_data(nla[NFTA_RULE_TABLE]),
3795                         nft_net->base_seq);
3796         audit_log_nfcfg(buf, info->nfmsg->nfgen_family, 1,
3797                         AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
3798         kfree(buf);
3799
3800         return nfnetlink_unicast(skb2, net, portid);
3801 }
3802
3803 void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
3804 {
3805         struct nft_expr *expr, *next;
3806
3807         /*
3808          * Careful: some expressions might not be initialized in case this
3809          * is called on error from nf_tables_newrule().
3810          */
3811         expr = nft_expr_first(rule);
3812         while (nft_expr_more(rule, expr)) {
3813                 next = nft_expr_next(expr);
3814                 nf_tables_expr_destroy(ctx, expr);
3815                 expr = next;
3816         }
3817         kfree(rule);
3818 }
3819
3820 static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3821 {
3822         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3823         nf_tables_rule_destroy(ctx, rule);
3824 }
3825
3826 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3827 {
3828         struct nft_expr *expr, *last;
3829         const struct nft_data *data;
3830         struct nft_rule *rule;
3831         int err;
3832
3833         if (ctx->level == NFT_JUMP_STACK_SIZE)
3834                 return -EMLINK;
3835
3836         list_for_each_entry(rule, &chain->rules, list) {
3837                 if (fatal_signal_pending(current))
3838                         return -EINTR;
3839
3840                 if (!nft_is_active_next(ctx->net, rule))
3841                         continue;
3842
3843                 nft_rule_for_each_expr(expr, last, rule) {
3844                         if (!expr->ops->validate)
3845                                 continue;
3846
3847                         err = expr->ops->validate(ctx, expr, &data);
3848                         if (err < 0)
3849                                 return err;
3850                 }
3851         }
3852
3853         return 0;
3854 }
3855 EXPORT_SYMBOL_GPL(nft_chain_validate);
3856
3857 static int nft_table_validate(struct net *net, const struct nft_table *table)
3858 {
3859         struct nft_chain *chain;
3860         struct nft_ctx ctx = {
3861                 .net    = net,
3862                 .family = table->family,
3863         };
3864         int err;
3865
3866         list_for_each_entry(chain, &table->chains, list) {
3867                 if (!nft_is_base_chain(chain))
3868                         continue;
3869
3870                 ctx.chain = chain;
3871                 err = nft_chain_validate(&ctx, chain);
3872                 if (err < 0)
3873                         return err;
3874
3875                 cond_resched();
3876         }
3877
3878         return 0;
3879 }
3880
3881 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
3882                          const struct nft_set_iter *iter,
3883                          struct nft_elem_priv *elem_priv)
3884 {
3885         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
3886         struct nft_ctx *pctx = (struct nft_ctx *)ctx;
3887         const struct nft_data *data;
3888         int err;
3889
3890         if (!nft_set_elem_active(ext, iter->genmask))
3891                 return 0;
3892
3893         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
3894             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
3895                 return 0;
3896
3897         data = nft_set_ext_data(ext);
3898         switch (data->verdict.code) {
3899         case NFT_JUMP:
3900         case NFT_GOTO:
3901                 pctx->level++;
3902                 err = nft_chain_validate(ctx, data->verdict.chain);
3903                 if (err < 0)
3904                         return err;
3905                 pctx->level--;
3906                 break;
3907         default:
3908                 break;
3909         }
3910
3911         return 0;
3912 }
3913
3914 int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
3915 {
3916         struct nft_set_iter dummy_iter = {
3917                 .genmask        = nft_genmask_next(ctx->net),
3918         };
3919         struct nft_set_elem_catchall *catchall;
3920
3921         struct nft_set_ext *ext;
3922         int ret = 0;
3923
3924         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
3925                 ext = nft_set_elem_ext(set, catchall->elem);
3926                 if (!nft_set_elem_active(ext, dummy_iter.genmask))
3927                         continue;
3928
3929                 ret = nft_setelem_validate(ctx, set, &dummy_iter, catchall->elem);
3930                 if (ret < 0)
3931                         return ret;
3932         }
3933
3934         return ret;
3935 }
3936
3937 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3938                                              const struct nft_chain *chain,
3939                                              const struct nlattr *nla);
3940
3941 #define NFT_RULE_MAXEXPRS       128
3942
3943 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3944                              const struct nlattr * const nla[])
3945 {
3946         struct nftables_pernet *nft_net = nft_pernet(info->net);
3947         struct netlink_ext_ack *extack = info->extack;
3948         unsigned int size, i, n, ulen = 0, usize = 0;
3949         u8 genmask = nft_genmask_next(info->net);
3950         struct nft_rule *rule, *old_rule = NULL;
3951         struct nft_expr_info *expr_info = NULL;
3952         u8 family = info->nfmsg->nfgen_family;
3953         struct nft_flow_rule *flow = NULL;
3954         struct net *net = info->net;
3955         struct nft_userdata *udata;
3956         struct nft_table *table;
3957         struct nft_chain *chain;
3958         struct nft_trans *trans;
3959         u64 handle, pos_handle;
3960         struct nft_expr *expr;
3961         struct nft_ctx ctx;
3962         struct nlattr *tmp;
3963         int err, rem;
3964
3965         lockdep_assert_held(&nft_net->commit_mutex);
3966
3967         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3968                                  NETLINK_CB(skb).portid);
3969         if (IS_ERR(table)) {
3970                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3971                 return PTR_ERR(table);
3972         }
3973
3974         if (nla[NFTA_RULE_CHAIN]) {
3975                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3976                                          genmask);
3977                 if (IS_ERR(chain)) {
3978                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3979                         return PTR_ERR(chain);
3980                 }
3981
3982         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3983                 chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID],
3984                                               genmask);
3985                 if (IS_ERR(chain)) {
3986                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3987                         return PTR_ERR(chain);
3988                 }
3989         } else {
3990                 return -EINVAL;
3991         }
3992
3993         if (nft_chain_is_bound(chain))
3994                 return -EOPNOTSUPP;
3995
3996         if (nla[NFTA_RULE_HANDLE]) {
3997                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3998                 rule = __nft_rule_lookup(chain, handle);
3999                 if (IS_ERR(rule)) {
4000                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4001                         return PTR_ERR(rule);
4002                 }
4003
4004                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
4005                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4006                         return -EEXIST;
4007                 }
4008                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4009                         old_rule = rule;
4010                 else
4011                         return -EOPNOTSUPP;
4012         } else {
4013                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
4014                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
4015                         return -EINVAL;
4016                 handle = nf_tables_alloc_handle(table);
4017
4018                 if (nla[NFTA_RULE_POSITION]) {
4019                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
4020                         old_rule = __nft_rule_lookup(chain, pos_handle);
4021                         if (IS_ERR(old_rule)) {
4022                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
4023                                 return PTR_ERR(old_rule);
4024                         }
4025                 } else if (nla[NFTA_RULE_POSITION_ID]) {
4026                         old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
4027                         if (IS_ERR(old_rule)) {
4028                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
4029                                 return PTR_ERR(old_rule);
4030                         }
4031                 }
4032         }
4033
4034         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4035
4036         n = 0;
4037         size = 0;
4038         if (nla[NFTA_RULE_EXPRESSIONS]) {
4039                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
4040                                            sizeof(struct nft_expr_info),
4041                                            GFP_KERNEL);
4042                 if (!expr_info)
4043                         return -ENOMEM;
4044
4045                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
4046                         err = -EINVAL;
4047                         if (nla_type(tmp) != NFTA_LIST_ELEM)
4048                                 goto err_release_expr;
4049                         if (n == NFT_RULE_MAXEXPRS)
4050                                 goto err_release_expr;
4051                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
4052                         if (err < 0) {
4053                                 NL_SET_BAD_ATTR(extack, tmp);
4054                                 goto err_release_expr;
4055                         }
4056                         size += expr_info[n].ops->size;
4057                         n++;
4058                 }
4059         }
4060         /* Check for overflow of dlen field */
4061         err = -EFBIG;
4062         if (size >= 1 << 12)
4063                 goto err_release_expr;
4064
4065         if (nla[NFTA_RULE_USERDATA]) {
4066                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
4067                 if (ulen > 0)
4068                         usize = sizeof(struct nft_userdata) + ulen;
4069         }
4070
4071         err = -ENOMEM;
4072         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
4073         if (rule == NULL)
4074                 goto err_release_expr;
4075
4076         nft_activate_next(net, rule);
4077
4078         rule->handle = handle;
4079         rule->dlen   = size;
4080         rule->udata  = ulen ? 1 : 0;
4081
4082         if (ulen) {
4083                 udata = nft_userdata(rule);
4084                 udata->len = ulen - 1;
4085                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
4086         }
4087
4088         expr = nft_expr_first(rule);
4089         for (i = 0; i < n; i++) {
4090                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
4091                 if (err < 0) {
4092                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
4093                         goto err_release_rule;
4094                 }
4095
4096                 if (expr_info[i].ops->validate)
4097                         nft_validate_state_update(table, NFT_VALIDATE_NEED);
4098
4099                 expr_info[i].ops = NULL;
4100                 expr = nft_expr_next(expr);
4101         }
4102
4103         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
4104                 flow = nft_flow_rule_create(net, rule);
4105                 if (IS_ERR(flow)) {
4106                         err = PTR_ERR(flow);
4107                         goto err_release_rule;
4108                 }
4109         }
4110
4111         if (!nft_use_inc(&chain->use)) {
4112                 err = -EMFILE;
4113                 goto err_release_rule;
4114         }
4115
4116         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
4117                 if (nft_chain_binding(chain)) {
4118                         err = -EOPNOTSUPP;
4119                         goto err_destroy_flow_rule;
4120                 }
4121
4122                 err = nft_delrule(&ctx, old_rule);
4123                 if (err < 0)
4124                         goto err_destroy_flow_rule;
4125
4126                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4127                 if (trans == NULL) {
4128                         err = -ENOMEM;
4129                         goto err_destroy_flow_rule;
4130                 }
4131                 list_add_tail_rcu(&rule->list, &old_rule->list);
4132         } else {
4133                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
4134                 if (!trans) {
4135                         err = -ENOMEM;
4136                         goto err_destroy_flow_rule;
4137                 }
4138
4139                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
4140                         if (old_rule)
4141                                 list_add_rcu(&rule->list, &old_rule->list);
4142                         else
4143                                 list_add_tail_rcu(&rule->list, &chain->rules);
4144                  } else {
4145                         if (old_rule)
4146                                 list_add_tail_rcu(&rule->list, &old_rule->list);
4147                         else
4148                                 list_add_rcu(&rule->list, &chain->rules);
4149                 }
4150         }
4151         kvfree(expr_info);
4152
4153         if (flow)
4154                 nft_trans_flow_rule(trans) = flow;
4155
4156         if (table->validate_state == NFT_VALIDATE_DO)
4157                 return nft_table_validate(net, table);
4158
4159         return 0;
4160
4161 err_destroy_flow_rule:
4162         nft_use_dec_restore(&chain->use);
4163         if (flow)
4164                 nft_flow_rule_destroy(flow);
4165 err_release_rule:
4166         nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
4167         nf_tables_rule_destroy(&ctx, rule);
4168 err_release_expr:
4169         for (i = 0; i < n; i++) {
4170                 if (expr_info[i].ops) {
4171                         module_put(expr_info[i].ops->type->owner);
4172                         if (expr_info[i].ops->type->release_ops)
4173                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
4174                 }
4175         }
4176         kvfree(expr_info);
4177
4178         return err;
4179 }
4180
4181 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
4182                                              const struct nft_chain *chain,
4183                                              const struct nlattr *nla)
4184 {
4185         struct nftables_pernet *nft_net = nft_pernet(net);
4186         u32 id = ntohl(nla_get_be32(nla));
4187         struct nft_trans *trans;
4188
4189         list_for_each_entry(trans, &nft_net->commit_list, list) {
4190                 if (trans->msg_type == NFT_MSG_NEWRULE &&
4191                     trans->ctx.chain == chain &&
4192                     id == nft_trans_rule_id(trans))
4193                         return nft_trans_rule(trans);
4194         }
4195         return ERR_PTR(-ENOENT);
4196 }
4197
4198 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
4199                              const struct nlattr * const nla[])
4200 {
4201         struct netlink_ext_ack *extack = info->extack;
4202         u8 genmask = nft_genmask_next(info->net);
4203         u8 family = info->nfmsg->nfgen_family;
4204         struct nft_chain *chain = NULL;
4205         struct net *net = info->net;
4206         struct nft_table *table;
4207         struct nft_rule *rule;
4208         struct nft_ctx ctx;
4209         int err = 0;
4210
4211         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
4212                                  NETLINK_CB(skb).portid);
4213         if (IS_ERR(table)) {
4214                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
4215                 return PTR_ERR(table);
4216         }
4217
4218         if (nla[NFTA_RULE_CHAIN]) {
4219                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
4220                                          genmask);
4221                 if (IS_ERR(chain)) {
4222                         if (PTR_ERR(chain) == -ENOENT &&
4223                             NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4224                                 return 0;
4225
4226                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
4227                         return PTR_ERR(chain);
4228                 }
4229                 if (nft_chain_binding(chain))
4230                         return -EOPNOTSUPP;
4231         }
4232
4233         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
4234
4235         if (chain) {
4236                 if (nla[NFTA_RULE_HANDLE]) {
4237                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
4238                         if (IS_ERR(rule)) {
4239                                 if (PTR_ERR(rule) == -ENOENT &&
4240                                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
4241                                         return 0;
4242
4243                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
4244                                 return PTR_ERR(rule);
4245                         }
4246
4247                         err = nft_delrule(&ctx, rule);
4248                 } else if (nla[NFTA_RULE_ID]) {
4249                         rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
4250                         if (IS_ERR(rule)) {
4251                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
4252                                 return PTR_ERR(rule);
4253                         }
4254
4255                         err = nft_delrule(&ctx, rule);
4256                 } else {
4257                         err = nft_delrule_by_chain(&ctx);
4258                 }
4259         } else {
4260                 list_for_each_entry(chain, &table->chains, list) {
4261                         if (!nft_is_active_next(net, chain))
4262                                 continue;
4263                         if (nft_chain_binding(chain))
4264                                 continue;
4265
4266                         ctx.chain = chain;
4267                         err = nft_delrule_by_chain(&ctx);
4268                         if (err < 0)
4269                                 break;
4270                 }
4271         }
4272
4273         return err;
4274 }
4275
4276 /*
4277  * Sets
4278  */
4279 static const struct nft_set_type *nft_set_types[] = {
4280         &nft_set_hash_fast_type,
4281         &nft_set_hash_type,
4282         &nft_set_rhash_type,
4283         &nft_set_bitmap_type,
4284         &nft_set_rbtree_type,
4285 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
4286         &nft_set_pipapo_avx2_type,
4287 #endif
4288         &nft_set_pipapo_type,
4289 };
4290
4291 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
4292                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
4293                                  NFT_SET_EVAL)
4294
4295 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
4296 {
4297         return (flags & type->features) == (flags & NFT_SET_FEATURES);
4298 }
4299
4300 /*
4301  * Select a set implementation based on the data characteristics and the
4302  * given policy. The total memory use might not be known if no size is
4303  * given, in that case the amount of memory per element is used.
4304  */
4305 static const struct nft_set_ops *
4306 nft_select_set_ops(const struct nft_ctx *ctx, u32 flags,
4307                    const struct nft_set_desc *desc)
4308 {
4309         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4310         const struct nft_set_ops *ops, *bops;
4311         struct nft_set_estimate est, best;
4312         const struct nft_set_type *type;
4313         int i;
4314
4315         lockdep_assert_held(&nft_net->commit_mutex);
4316         lockdep_nfnl_nft_mutex_not_held();
4317
4318         bops        = NULL;
4319         best.size   = ~0;
4320         best.lookup = ~0;
4321         best.space  = ~0;
4322
4323         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
4324                 type = nft_set_types[i];
4325                 ops = &type->ops;
4326
4327                 if (!nft_set_ops_candidate(type, flags))
4328                         continue;
4329                 if (!ops->estimate(desc, flags, &est))
4330                         continue;
4331
4332                 switch (desc->policy) {
4333                 case NFT_SET_POL_PERFORMANCE:
4334                         if (est.lookup < best.lookup)
4335                                 break;
4336                         if (est.lookup == best.lookup &&
4337                             est.space < best.space)
4338                                 break;
4339                         continue;
4340                 case NFT_SET_POL_MEMORY:
4341                         if (!desc->size) {
4342                                 if (est.space < best.space)
4343                                         break;
4344                                 if (est.space == best.space &&
4345                                     est.lookup < best.lookup)
4346                                         break;
4347                         } else if (est.size < best.size || !bops) {
4348                                 break;
4349                         }
4350                         continue;
4351                 default:
4352                         break;
4353                 }
4354
4355                 bops = ops;
4356                 best = est;
4357         }
4358
4359         if (bops != NULL)
4360                 return bops;
4361
4362         return ERR_PTR(-EOPNOTSUPP);
4363 }
4364
4365 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
4366         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
4367                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4368         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
4369                                             .len = NFT_SET_MAXNAMELEN - 1 },
4370         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
4371         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
4372         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
4373         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
4374         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
4375         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
4376         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
4377         [NFTA_SET_ID]                   = { .type = NLA_U32 },
4378         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
4379         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
4380         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
4381                                             .len  = NFT_USERDATA_MAXLEN },
4382         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
4383         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
4384         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
4385         [NFTA_SET_EXPRESSIONS]          = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
4386 };
4387
4388 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4389         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4390 };
4391
4392 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
4393         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
4394         [NFTA_SET_DESC_CONCAT]          = NLA_POLICY_NESTED_ARRAY(nft_concat_policy),
4395 };
4396
4397 static struct nft_set *nft_set_lookup(const struct nft_table *table,
4398                                       const struct nlattr *nla, u8 genmask)
4399 {
4400         struct nft_set *set;
4401
4402         if (nla == NULL)
4403                 return ERR_PTR(-EINVAL);
4404
4405         list_for_each_entry_rcu(set, &table->sets, list) {
4406                 if (!nla_strcmp(nla, set->name) &&
4407                     nft_active_genmask(set, genmask))
4408                         return set;
4409         }
4410         return ERR_PTR(-ENOENT);
4411 }
4412
4413 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
4414                                                const struct nlattr *nla,
4415                                                u8 genmask)
4416 {
4417         struct nft_set *set;
4418
4419         list_for_each_entry(set, &table->sets, list) {
4420                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
4421                     nft_active_genmask(set, genmask))
4422                         return set;
4423         }
4424         return ERR_PTR(-ENOENT);
4425 }
4426
4427 static struct nft_set *nft_set_lookup_byid(const struct net *net,
4428                                            const struct nft_table *table,
4429                                            const struct nlattr *nla, u8 genmask)
4430 {
4431         struct nftables_pernet *nft_net = nft_pernet(net);
4432         u32 id = ntohl(nla_get_be32(nla));
4433         struct nft_trans *trans;
4434
4435         list_for_each_entry(trans, &nft_net->commit_list, list) {
4436                 if (trans->msg_type == NFT_MSG_NEWSET) {
4437                         struct nft_set *set = nft_trans_set(trans);
4438
4439                         if (id == nft_trans_set_id(trans) &&
4440                             set->table == table &&
4441                             nft_active_genmask(set, genmask))
4442                                 return set;
4443                 }
4444         }
4445         return ERR_PTR(-ENOENT);
4446 }
4447
4448 struct nft_set *nft_set_lookup_global(const struct net *net,
4449                                       const struct nft_table *table,
4450                                       const struct nlattr *nla_set_name,
4451                                       const struct nlattr *nla_set_id,
4452                                       u8 genmask)
4453 {
4454         struct nft_set *set;
4455
4456         set = nft_set_lookup(table, nla_set_name, genmask);
4457         if (IS_ERR(set)) {
4458                 if (!nla_set_id)
4459                         return set;
4460
4461                 set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
4462         }
4463         return set;
4464 }
4465 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
4466
4467 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
4468                                     const char *name)
4469 {
4470         const struct nft_set *i;
4471         const char *p;
4472         unsigned long *inuse;
4473         unsigned int n = 0, min = 0;
4474
4475         p = strchr(name, '%');
4476         if (p != NULL) {
4477                 if (p[1] != 'd' || strchr(p + 2, '%'))
4478                         return -EINVAL;
4479
4480                 if (strnlen(name, NFT_SET_MAX_ANONLEN) >= NFT_SET_MAX_ANONLEN)
4481                         return -EINVAL;
4482
4483                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
4484                 if (inuse == NULL)
4485                         return -ENOMEM;
4486 cont:
4487                 list_for_each_entry(i, &ctx->table->sets, list) {
4488                         int tmp;
4489
4490                         if (!nft_is_active_next(ctx->net, i))
4491                                 continue;
4492                         if (!sscanf(i->name, name, &tmp))
4493                                 continue;
4494                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
4495                                 continue;
4496
4497                         set_bit(tmp - min, inuse);
4498                 }
4499
4500                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
4501                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
4502                         min += BITS_PER_BYTE * PAGE_SIZE;
4503                         memset(inuse, 0, PAGE_SIZE);
4504                         goto cont;
4505                 }
4506                 free_page((unsigned long)inuse);
4507         }
4508
4509         set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
4510         if (!set->name)
4511                 return -ENOMEM;
4512
4513         list_for_each_entry(i, &ctx->table->sets, list) {
4514                 if (!nft_is_active_next(ctx->net, i))
4515                         continue;
4516                 if (!strcmp(set->name, i->name)) {
4517                         kfree(set->name);
4518                         set->name = NULL;
4519                         return -ENFILE;
4520                 }
4521         }
4522         return 0;
4523 }
4524
4525 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
4526 {
4527         u64 ms = be64_to_cpu(nla_get_be64(nla));
4528         u64 max = (u64)(~((u64)0));
4529
4530         max = div_u64(max, NSEC_PER_MSEC);
4531         if (ms >= max)
4532                 return -ERANGE;
4533
4534         ms *= NSEC_PER_MSEC;
4535         *result = nsecs_to_jiffies64(ms);
4536         return 0;
4537 }
4538
4539 __be64 nf_jiffies64_to_msecs(u64 input)
4540 {
4541         return cpu_to_be64(jiffies64_to_msecs(input));
4542 }
4543
4544 static int nf_tables_fill_set_concat(struct sk_buff *skb,
4545                                      const struct nft_set *set)
4546 {
4547         struct nlattr *concat, *field;
4548         int i;
4549
4550         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
4551         if (!concat)
4552                 return -ENOMEM;
4553
4554         for (i = 0; i < set->field_count; i++) {
4555                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4556                 if (!field)
4557                         return -ENOMEM;
4558
4559                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
4560                                  htonl(set->field_len[i])))
4561                         return -ENOMEM;
4562
4563                 nla_nest_end(skb, field);
4564         }
4565
4566         nla_nest_end(skb, concat);
4567
4568         return 0;
4569 }
4570
4571 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
4572                               const struct nft_set *set, u16 event, u16 flags)
4573 {
4574         u64 timeout = READ_ONCE(set->timeout);
4575         u32 gc_int = READ_ONCE(set->gc_int);
4576         u32 portid = ctx->portid;
4577         struct nlmsghdr *nlh;
4578         struct nlattr *nest;
4579         u32 seq = ctx->seq;
4580         int i;
4581
4582         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4583         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4584                            NFNETLINK_V0, nft_base_seq(ctx->net));
4585         if (!nlh)
4586                 goto nla_put_failure;
4587
4588         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4589                 goto nla_put_failure;
4590         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4591                 goto nla_put_failure;
4592         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
4593                          NFTA_SET_PAD))
4594                 goto nla_put_failure;
4595
4596         if (event == NFT_MSG_DELSET) {
4597                 nlmsg_end(skb, nlh);
4598                 return 0;
4599         }
4600
4601         if (set->flags != 0)
4602                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
4603                         goto nla_put_failure;
4604
4605         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
4606                 goto nla_put_failure;
4607         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
4608                 goto nla_put_failure;
4609         if (set->flags & NFT_SET_MAP) {
4610                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
4611                         goto nla_put_failure;
4612                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
4613                         goto nla_put_failure;
4614         }
4615         if (set->flags & NFT_SET_OBJECT &&
4616             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
4617                 goto nla_put_failure;
4618
4619         if (timeout &&
4620             nla_put_be64(skb, NFTA_SET_TIMEOUT,
4621                          nf_jiffies64_to_msecs(timeout),
4622                          NFTA_SET_PAD))
4623                 goto nla_put_failure;
4624         if (gc_int &&
4625             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
4626                 goto nla_put_failure;
4627
4628         if (set->policy != NFT_SET_POL_PERFORMANCE) {
4629                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
4630                         goto nla_put_failure;
4631         }
4632
4633         if (set->udata &&
4634             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
4635                 goto nla_put_failure;
4636
4637         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
4638         if (!nest)
4639                 goto nla_put_failure;
4640         if (set->size &&
4641             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
4642                 goto nla_put_failure;
4643
4644         if (set->field_count > 1 &&
4645             nf_tables_fill_set_concat(skb, set))
4646                 goto nla_put_failure;
4647
4648         nla_nest_end(skb, nest);
4649
4650         if (set->num_exprs == 1) {
4651                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
4652                 if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
4653                         goto nla_put_failure;
4654
4655                 nla_nest_end(skb, nest);
4656         } else if (set->num_exprs > 1) {
4657                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
4658                 if (nest == NULL)
4659                         goto nla_put_failure;
4660
4661                 for (i = 0; i < set->num_exprs; i++) {
4662                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
4663                                           set->exprs[i], false) < 0)
4664                                 goto nla_put_failure;
4665                 }
4666                 nla_nest_end(skb, nest);
4667         }
4668
4669         nlmsg_end(skb, nlh);
4670         return 0;
4671
4672 nla_put_failure:
4673         nlmsg_trim(skb, nlh);
4674         return -1;
4675 }
4676
4677 static void nf_tables_set_notify(const struct nft_ctx *ctx,
4678                                  const struct nft_set *set, int event,
4679                                  gfp_t gfp_flags)
4680 {
4681         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
4682         u32 portid = ctx->portid;
4683         struct sk_buff *skb;
4684         u16 flags = 0;
4685         int err;
4686
4687         if (!ctx->report &&
4688             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
4689                 return;
4690
4691         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
4692         if (skb == NULL)
4693                 goto err;
4694
4695         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
4696                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
4697
4698         err = nf_tables_fill_set(skb, ctx, set, event, flags);
4699         if (err < 0) {
4700                 kfree_skb(skb);
4701                 goto err;
4702         }
4703
4704         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
4705         return;
4706 err:
4707         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
4708 }
4709
4710 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
4711 {
4712         const struct nft_set *set;
4713         unsigned int idx, s_idx = cb->args[0];
4714         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
4715         struct net *net = sock_net(skb->sk);
4716         struct nft_ctx *ctx = cb->data, ctx_set;
4717         struct nftables_pernet *nft_net;
4718
4719         if (cb->args[1])
4720                 return skb->len;
4721
4722         rcu_read_lock();
4723         nft_net = nft_pernet(net);
4724         cb->seq = READ_ONCE(nft_net->base_seq);
4725
4726         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4727                 if (ctx->family != NFPROTO_UNSPEC &&
4728                     ctx->family != table->family)
4729                         continue;
4730
4731                 if (ctx->table && ctx->table != table)
4732                         continue;
4733
4734                 if (cur_table) {
4735                         if (cur_table != table)
4736                                 continue;
4737
4738                         cur_table = NULL;
4739                 }
4740                 idx = 0;
4741                 list_for_each_entry_rcu(set, &table->sets, list) {
4742                         if (idx < s_idx)
4743                                 goto cont;
4744                         if (!nft_is_active(net, set))
4745                                 goto cont;
4746
4747                         ctx_set = *ctx;
4748                         ctx_set.table = table;
4749                         ctx_set.family = table->family;
4750
4751                         if (nf_tables_fill_set(skb, &ctx_set, set,
4752                                                NFT_MSG_NEWSET,
4753                                                NLM_F_MULTI) < 0) {
4754                                 cb->args[0] = idx;
4755                                 cb->args[2] = (unsigned long) table;
4756                                 goto done;
4757                         }
4758                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4759 cont:
4760                         idx++;
4761                 }
4762                 if (s_idx)
4763                         s_idx = 0;
4764         }
4765         cb->args[1] = 1;
4766 done:
4767         rcu_read_unlock();
4768         return skb->len;
4769 }
4770
4771 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4772 {
4773         struct nft_ctx *ctx_dump = NULL;
4774
4775         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4776         if (ctx_dump == NULL)
4777                 return -ENOMEM;
4778
4779         cb->data = ctx_dump;
4780         return 0;
4781 }
4782
4783 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4784 {
4785         kfree(cb->data);
4786         return 0;
4787 }
4788
4789 /* called with rcu_read_lock held */
4790 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4791                             const struct nlattr * const nla[])
4792 {
4793         struct netlink_ext_ack *extack = info->extack;
4794         u8 genmask = nft_genmask_cur(info->net);
4795         u8 family = info->nfmsg->nfgen_family;
4796         struct nft_table *table = NULL;
4797         struct net *net = info->net;
4798         const struct nft_set *set;
4799         struct sk_buff *skb2;
4800         struct nft_ctx ctx;
4801         int err;
4802
4803         if (nla[NFTA_SET_TABLE]) {
4804                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4805                                          genmask, 0);
4806                 if (IS_ERR(table)) {
4807                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4808                         return PTR_ERR(table);
4809                 }
4810         }
4811
4812         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4813
4814         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4815                 struct netlink_dump_control c = {
4816                         .start = nf_tables_dump_sets_start,
4817                         .dump = nf_tables_dump_sets,
4818                         .done = nf_tables_dump_sets_done,
4819                         .data = &ctx,
4820                         .module = THIS_MODULE,
4821                 };
4822
4823                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4824         }
4825
4826         /* Only accept unspec with dump */
4827         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4828                 return -EAFNOSUPPORT;
4829         if (!nla[NFTA_SET_TABLE])
4830                 return -EINVAL;
4831
4832         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4833         if (IS_ERR(set)) {
4834                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4835                 return PTR_ERR(set);
4836         }
4837
4838         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4839         if (skb2 == NULL)
4840                 return -ENOMEM;
4841
4842         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4843         if (err < 0)
4844                 goto err_fill_set_info;
4845
4846         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4847
4848 err_fill_set_info:
4849         kfree_skb(skb2);
4850         return err;
4851 }
4852
4853 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4854                                      struct nft_set_desc *desc)
4855 {
4856         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4857         u32 len;
4858         int err;
4859
4860         if (desc->field_count >= ARRAY_SIZE(desc->field_len))
4861                 return -E2BIG;
4862
4863         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4864                                           nft_concat_policy, NULL);
4865         if (err < 0)
4866                 return err;
4867
4868         if (!tb[NFTA_SET_FIELD_LEN])
4869                 return -EINVAL;
4870
4871         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4872         if (!len || len > U8_MAX)
4873                 return -EINVAL;
4874
4875         desc->field_len[desc->field_count++] = len;
4876
4877         return 0;
4878 }
4879
4880 static int nft_set_desc_concat(struct nft_set_desc *desc,
4881                                const struct nlattr *nla)
4882 {
4883         u32 num_regs = 0, key_num_regs = 0;
4884         struct nlattr *attr;
4885         int rem, err, i;
4886
4887         nla_for_each_nested(attr, nla, rem) {
4888                 if (nla_type(attr) != NFTA_LIST_ELEM)
4889                         return -EINVAL;
4890
4891                 err = nft_set_desc_concat_parse(attr, desc);
4892                 if (err < 0)
4893                         return err;
4894         }
4895
4896         for (i = 0; i < desc->field_count; i++)
4897                 num_regs += DIV_ROUND_UP(desc->field_len[i], sizeof(u32));
4898
4899         key_num_regs = DIV_ROUND_UP(desc->klen, sizeof(u32));
4900         if (key_num_regs != num_regs)
4901                 return -EINVAL;
4902
4903         if (num_regs > NFT_REG32_COUNT)
4904                 return -E2BIG;
4905
4906         return 0;
4907 }
4908
4909 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4910                                     const struct nlattr *nla)
4911 {
4912         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4913         int err;
4914
4915         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4916                                           nft_set_desc_policy, NULL);
4917         if (err < 0)
4918                 return err;
4919
4920         if (da[NFTA_SET_DESC_SIZE] != NULL)
4921                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4922         if (da[NFTA_SET_DESC_CONCAT])
4923                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4924
4925         return err;
4926 }
4927
4928 static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
4929                               const struct nlattr * const *nla,
4930                               struct nft_expr **exprs, int *num_exprs,
4931                               u32 flags)
4932 {
4933         struct nft_expr *expr;
4934         int err, i;
4935
4936         if (nla[NFTA_SET_EXPR]) {
4937                 expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
4938                 if (IS_ERR(expr)) {
4939                         err = PTR_ERR(expr);
4940                         goto err_set_expr_alloc;
4941                 }
4942                 exprs[0] = expr;
4943                 (*num_exprs)++;
4944         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4945                 struct nlattr *tmp;
4946                 int left;
4947
4948                 if (!(flags & NFT_SET_EXPR)) {
4949                         err = -EINVAL;
4950                         goto err_set_expr_alloc;
4951                 }
4952                 i = 0;
4953                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4954                         if (i == NFT_SET_EXPR_MAX) {
4955                                 err = -E2BIG;
4956                                 goto err_set_expr_alloc;
4957                         }
4958                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4959                                 err = -EINVAL;
4960                                 goto err_set_expr_alloc;
4961                         }
4962                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
4963                         if (IS_ERR(expr)) {
4964                                 err = PTR_ERR(expr);
4965                                 goto err_set_expr_alloc;
4966                         }
4967                         exprs[i++] = expr;
4968                         (*num_exprs)++;
4969                 }
4970         }
4971
4972         return 0;
4973
4974 err_set_expr_alloc:
4975         for (i = 0; i < *num_exprs; i++)
4976                 nft_expr_destroy(ctx, exprs[i]);
4977
4978         return err;
4979 }
4980
4981 static bool nft_set_is_same(const struct nft_set *set,
4982                             const struct nft_set_desc *desc,
4983                             struct nft_expr *exprs[], u32 num_exprs, u32 flags)
4984 {
4985         int i;
4986
4987         if (set->ktype != desc->ktype ||
4988             set->dtype != desc->dtype ||
4989             set->flags != flags ||
4990             set->klen != desc->klen ||
4991             set->dlen != desc->dlen ||
4992             set->field_count != desc->field_count ||
4993             set->num_exprs != num_exprs)
4994                 return false;
4995
4996         for (i = 0; i < desc->field_count; i++) {
4997                 if (set->field_len[i] != desc->field_len[i])
4998                         return false;
4999         }
5000
5001         for (i = 0; i < num_exprs; i++) {
5002                 if (set->exprs[i]->ops != exprs[i]->ops)
5003                         return false;
5004         }
5005
5006         return true;
5007 }
5008
5009 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
5010                             const struct nlattr * const nla[])
5011 {
5012         struct netlink_ext_ack *extack = info->extack;
5013         u8 genmask = nft_genmask_next(info->net);
5014         u8 family = info->nfmsg->nfgen_family;
5015         const struct nft_set_ops *ops;
5016         struct net *net = info->net;
5017         struct nft_set_desc desc;
5018         struct nft_table *table;
5019         unsigned char *udata;
5020         struct nft_set *set;
5021         struct nft_ctx ctx;
5022         size_t alloc_size;
5023         int num_exprs = 0;
5024         char *name;
5025         int err, i;
5026         u16 udlen;
5027         u32 flags;
5028         u64 size;
5029
5030         if (nla[NFTA_SET_TABLE] == NULL ||
5031             nla[NFTA_SET_NAME] == NULL ||
5032             nla[NFTA_SET_KEY_LEN] == NULL ||
5033             nla[NFTA_SET_ID] == NULL)
5034                 return -EINVAL;
5035
5036         memset(&desc, 0, sizeof(desc));
5037
5038         desc.ktype = NFT_DATA_VALUE;
5039         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
5040                 desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
5041                 if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
5042                         return -EINVAL;
5043         }
5044
5045         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
5046         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
5047                 return -EINVAL;
5048
5049         flags = 0;
5050         if (nla[NFTA_SET_FLAGS] != NULL) {
5051                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
5052                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
5053                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
5054                               NFT_SET_MAP | NFT_SET_EVAL |
5055                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
5056                         return -EOPNOTSUPP;
5057                 /* Only one of these operations is supported */
5058                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
5059                              (NFT_SET_MAP | NFT_SET_OBJECT))
5060                         return -EOPNOTSUPP;
5061                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
5062                              (NFT_SET_EVAL | NFT_SET_OBJECT))
5063                         return -EOPNOTSUPP;
5064                 if ((flags & (NFT_SET_ANONYMOUS | NFT_SET_TIMEOUT | NFT_SET_EVAL)) ==
5065                              (NFT_SET_ANONYMOUS | NFT_SET_TIMEOUT))
5066                         return -EOPNOTSUPP;
5067                 if ((flags & (NFT_SET_CONSTANT | NFT_SET_TIMEOUT)) ==
5068                              (NFT_SET_CONSTANT | NFT_SET_TIMEOUT))
5069                         return -EOPNOTSUPP;
5070         }
5071
5072         desc.dtype = 0;
5073         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
5074                 if (!(flags & NFT_SET_MAP))
5075                         return -EINVAL;
5076
5077                 desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
5078                 if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
5079                     desc.dtype != NFT_DATA_VERDICT)
5080                         return -EINVAL;
5081
5082                 if (desc.dtype != NFT_DATA_VERDICT) {
5083                         if (nla[NFTA_SET_DATA_LEN] == NULL)
5084                                 return -EINVAL;
5085                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
5086                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
5087                                 return -EINVAL;
5088                 } else
5089                         desc.dlen = sizeof(struct nft_verdict);
5090         } else if (flags & NFT_SET_MAP)
5091                 return -EINVAL;
5092
5093         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
5094                 if (!(flags & NFT_SET_OBJECT))
5095                         return -EINVAL;
5096
5097                 desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
5098                 if (desc.objtype == NFT_OBJECT_UNSPEC ||
5099                     desc.objtype > NFT_OBJECT_MAX)
5100                         return -EOPNOTSUPP;
5101         } else if (flags & NFT_SET_OBJECT)
5102                 return -EINVAL;
5103         else
5104                 desc.objtype = NFT_OBJECT_UNSPEC;
5105
5106         desc.timeout = 0;
5107         if (nla[NFTA_SET_TIMEOUT] != NULL) {
5108                 if (!(flags & NFT_SET_TIMEOUT))
5109                         return -EINVAL;
5110
5111                 if (flags & NFT_SET_ANONYMOUS)
5112                         return -EOPNOTSUPP;
5113
5114                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
5115                 if (err)
5116                         return err;
5117         }
5118         desc.gc_int = 0;
5119         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
5120                 if (!(flags & NFT_SET_TIMEOUT))
5121                         return -EINVAL;
5122
5123                 if (flags & NFT_SET_ANONYMOUS)
5124                         return -EOPNOTSUPP;
5125
5126                 desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
5127         }
5128
5129         desc.policy = NFT_SET_POL_PERFORMANCE;
5130         if (nla[NFTA_SET_POLICY] != NULL) {
5131                 desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
5132                 switch (desc.policy) {
5133                 case NFT_SET_POL_PERFORMANCE:
5134                 case NFT_SET_POL_MEMORY:
5135                         break;
5136                 default:
5137                         return -EOPNOTSUPP;
5138                 }
5139         }
5140
5141         if (nla[NFTA_SET_DESC] != NULL) {
5142                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
5143                 if (err < 0)
5144                         return err;
5145
5146                 if (desc.field_count > 1) {
5147                         if (!(flags & NFT_SET_CONCAT))
5148                                 return -EINVAL;
5149                 } else if (flags & NFT_SET_CONCAT) {
5150                         return -EINVAL;
5151                 }
5152         } else if (flags & NFT_SET_CONCAT) {
5153                 return -EINVAL;
5154         }
5155
5156         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
5157                 desc.expr = true;
5158
5159         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
5160                                  NETLINK_CB(skb).portid);
5161         if (IS_ERR(table)) {
5162                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5163                 return PTR_ERR(table);
5164         }
5165
5166         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5167
5168         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
5169         if (IS_ERR(set)) {
5170                 if (PTR_ERR(set) != -ENOENT) {
5171                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5172                         return PTR_ERR(set);
5173                 }
5174         } else {
5175                 struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
5176
5177                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
5178                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5179                         return -EEXIST;
5180                 }
5181                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
5182                         return -EOPNOTSUPP;
5183
5184                 if (nft_set_is_anonymous(set))
5185                         return -EOPNOTSUPP;
5186
5187                 err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
5188                 if (err < 0)
5189                         return err;
5190
5191                 err = 0;
5192                 if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
5193                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
5194                         err = -EEXIST;
5195                 }
5196
5197                 for (i = 0; i < num_exprs; i++)
5198                         nft_expr_destroy(&ctx, exprs[i]);
5199
5200                 if (err < 0)
5201                         return err;
5202
5203                 return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
5204         }
5205
5206         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
5207                 return -ENOENT;
5208
5209         ops = nft_select_set_ops(&ctx, flags, &desc);
5210         if (IS_ERR(ops))
5211                 return PTR_ERR(ops);
5212
5213         udlen = 0;
5214         if (nla[NFTA_SET_USERDATA])
5215                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
5216
5217         size = 0;
5218         if (ops->privsize != NULL)
5219                 size = ops->privsize(nla, &desc);
5220         alloc_size = sizeof(*set) + size + udlen;
5221         if (alloc_size < size || alloc_size > INT_MAX)
5222                 return -ENOMEM;
5223
5224         if (!nft_use_inc(&table->use))
5225                 return -EMFILE;
5226
5227         set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
5228         if (!set) {
5229                 err = -ENOMEM;
5230                 goto err_alloc;
5231         }
5232
5233         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
5234         if (!name) {
5235                 err = -ENOMEM;
5236                 goto err_set_name;
5237         }
5238
5239         err = nf_tables_set_alloc_name(&ctx, set, name);
5240         kfree(name);
5241         if (err < 0)
5242                 goto err_set_name;
5243
5244         udata = NULL;
5245         if (udlen) {
5246                 udata = set->data + size;
5247                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
5248         }
5249
5250         INIT_LIST_HEAD(&set->bindings);
5251         INIT_LIST_HEAD(&set->catchall_list);
5252         refcount_set(&set->refs, 1);
5253         set->table = table;
5254         write_pnet(&set->net, net);
5255         set->ops = ops;
5256         set->ktype = desc.ktype;
5257         set->klen = desc.klen;
5258         set->dtype = desc.dtype;
5259         set->objtype = desc.objtype;
5260         set->dlen = desc.dlen;
5261         set->flags = flags;
5262         set->size = desc.size;
5263         set->policy = desc.policy;
5264         set->udlen = udlen;
5265         set->udata = udata;
5266         set->timeout = desc.timeout;
5267         set->gc_int = desc.gc_int;
5268
5269         set->field_count = desc.field_count;
5270         for (i = 0; i < desc.field_count; i++)
5271                 set->field_len[i] = desc.field_len[i];
5272
5273         err = ops->init(set, &desc, nla);
5274         if (err < 0)
5275                 goto err_set_init;
5276
5277         err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
5278         if (err < 0)
5279                 goto err_set_destroy;
5280
5281         set->num_exprs = num_exprs;
5282         set->handle = nf_tables_alloc_handle(table);
5283         INIT_LIST_HEAD(&set->pending_update);
5284
5285         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
5286         if (err < 0)
5287                 goto err_set_expr_alloc;
5288
5289         list_add_tail_rcu(&set->list, &table->sets);
5290
5291         return 0;
5292
5293 err_set_expr_alloc:
5294         for (i = 0; i < set->num_exprs; i++)
5295                 nft_expr_destroy(&ctx, set->exprs[i]);
5296 err_set_destroy:
5297         ops->destroy(&ctx, set);
5298 err_set_init:
5299         kfree(set->name);
5300 err_set_name:
5301         kvfree(set);
5302 err_alloc:
5303         nft_use_dec_restore(&table->use);
5304
5305         return err;
5306 }
5307
5308 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
5309                                      struct nft_set *set)
5310 {
5311         struct nft_set_elem_catchall *next, *catchall;
5312
5313         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5314                 list_del_rcu(&catchall->list);
5315                 nf_tables_set_elem_destroy(ctx, set, catchall->elem);
5316                 kfree_rcu(catchall, rcu);
5317         }
5318 }
5319
5320 static void nft_set_put(struct nft_set *set)
5321 {
5322         if (refcount_dec_and_test(&set->refs)) {
5323                 kfree(set->name);
5324                 kvfree(set);
5325         }
5326 }
5327
5328 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5329 {
5330         int i;
5331
5332         if (WARN_ON(set->use > 0))
5333                 return;
5334
5335         for (i = 0; i < set->num_exprs; i++)
5336                 nft_expr_destroy(ctx, set->exprs[i]);
5337
5338         set->ops->destroy(ctx, set);
5339         nft_set_catchall_destroy(ctx, set);
5340         nft_set_put(set);
5341 }
5342
5343 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
5344                             const struct nlattr * const nla[])
5345 {
5346         struct netlink_ext_ack *extack = info->extack;
5347         u8 genmask = nft_genmask_next(info->net);
5348         u8 family = info->nfmsg->nfgen_family;
5349         struct net *net = info->net;
5350         const struct nlattr *attr;
5351         struct nft_table *table;
5352         struct nft_set *set;
5353         struct nft_ctx ctx;
5354
5355         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
5356                 return -EAFNOSUPPORT;
5357
5358         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
5359                                  genmask, NETLINK_CB(skb).portid);
5360         if (IS_ERR(table)) {
5361                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
5362                 return PTR_ERR(table);
5363         }
5364
5365         if (nla[NFTA_SET_HANDLE]) {
5366                 attr = nla[NFTA_SET_HANDLE];
5367                 set = nft_set_lookup_byhandle(table, attr, genmask);
5368         } else {
5369                 attr = nla[NFTA_SET_NAME];
5370                 set = nft_set_lookup(table, attr, genmask);
5371         }
5372
5373         if (IS_ERR(set)) {
5374                 if (PTR_ERR(set) == -ENOENT &&
5375                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
5376                         return 0;
5377
5378                 NL_SET_BAD_ATTR(extack, attr);
5379                 return PTR_ERR(set);
5380         }
5381         if (set->use ||
5382             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
5383              atomic_read(&set->nelems) > 0)) {
5384                 NL_SET_BAD_ATTR(extack, attr);
5385                 return -EBUSY;
5386         }
5387
5388         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5389
5390         return nft_delset(&ctx, set);
5391 }
5392
5393 static int nft_validate_register_store(const struct nft_ctx *ctx,
5394                                        enum nft_registers reg,
5395                                        const struct nft_data *data,
5396                                        enum nft_data_types type,
5397                                        unsigned int len);
5398
5399 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
5400                                      struct nft_set *set,
5401                                      struct nft_elem_priv *elem_priv)
5402 {
5403         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5404         enum nft_registers dreg;
5405
5406         dreg = nft_type_to_reg(set->dtype);
5407         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
5408                                            set->dtype == NFT_DATA_VERDICT ?
5409                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
5410                                            set->dlen);
5411 }
5412
5413 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
5414                                         struct nft_set *set,
5415                                         const struct nft_set_iter *iter,
5416                                         struct nft_elem_priv *elem_priv)
5417 {
5418         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5419
5420         if (!nft_set_elem_active(ext, iter->genmask))
5421                 return 0;
5422
5423         return nft_setelem_data_validate(ctx, set, elem_priv);
5424 }
5425
5426 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
5427                                        struct nft_set *set)
5428 {
5429         u8 genmask = nft_genmask_next(ctx->net);
5430         struct nft_set_elem_catchall *catchall;
5431         struct nft_set_ext *ext;
5432         int ret = 0;
5433
5434         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5435                 ext = nft_set_elem_ext(set, catchall->elem);
5436                 if (!nft_set_elem_active(ext, genmask))
5437                         continue;
5438
5439                 ret = nft_setelem_data_validate(ctx, set, catchall->elem);
5440                 if (ret < 0)
5441                         break;
5442         }
5443
5444         return ret;
5445 }
5446
5447 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
5448                        struct nft_set_binding *binding)
5449 {
5450         struct nft_set_binding *i;
5451         struct nft_set_iter iter;
5452
5453         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
5454                 return -EBUSY;
5455
5456         if (binding->flags & NFT_SET_MAP) {
5457                 /* If the set is already bound to the same chain all
5458                  * jumps are already validated for that chain.
5459                  */
5460                 list_for_each_entry(i, &set->bindings, list) {
5461                         if (i->flags & NFT_SET_MAP &&
5462                             i->chain == binding->chain)
5463                                 goto bind;
5464                 }
5465
5466                 iter.genmask    = nft_genmask_next(ctx->net);
5467                 iter.type       = NFT_ITER_UPDATE;
5468                 iter.skip       = 0;
5469                 iter.count      = 0;
5470                 iter.err        = 0;
5471                 iter.fn         = nf_tables_bind_check_setelem;
5472
5473                 set->ops->walk(ctx, set, &iter);
5474                 if (!iter.err)
5475                         iter.err = nft_set_catchall_bind_check(ctx, set);
5476
5477                 if (iter.err < 0)
5478                         return iter.err;
5479         }
5480 bind:
5481         if (!nft_use_inc(&set->use))
5482                 return -EMFILE;
5483
5484         binding->chain = ctx->chain;
5485         list_add_tail_rcu(&binding->list, &set->bindings);
5486         nft_set_trans_bind(ctx, set);
5487
5488         return 0;
5489 }
5490 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
5491
5492 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
5493                                  struct nft_set_binding *binding, bool event)
5494 {
5495         list_del_rcu(&binding->list);
5496
5497         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
5498                 list_del_rcu(&set->list);
5499                 set->dead = 1;
5500                 if (event)
5501                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
5502                                              GFP_KERNEL);
5503         }
5504 }
5505
5506 static void nft_setelem_data_activate(const struct net *net,
5507                                       const struct nft_set *set,
5508                                       struct nft_elem_priv *elem_priv);
5509
5510 static int nft_mapelem_activate(const struct nft_ctx *ctx,
5511                                 struct nft_set *set,
5512                                 const struct nft_set_iter *iter,
5513                                 struct nft_elem_priv *elem_priv)
5514 {
5515         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5516
5517         /* called from abort path, reverse check to undo changes. */
5518         if (nft_set_elem_active(ext, iter->genmask))
5519                 return 0;
5520
5521         nft_clear(ctx->net, ext);
5522         nft_setelem_data_activate(ctx->net, set, elem_priv);
5523
5524         return 0;
5525 }
5526
5527 static void nft_map_catchall_activate(const struct nft_ctx *ctx,
5528                                       struct nft_set *set)
5529 {
5530         u8 genmask = nft_genmask_next(ctx->net);
5531         struct nft_set_elem_catchall *catchall;
5532         struct nft_set_ext *ext;
5533
5534         list_for_each_entry(catchall, &set->catchall_list, list) {
5535                 ext = nft_set_elem_ext(set, catchall->elem);
5536                 if (!nft_set_elem_active(ext, genmask))
5537                         continue;
5538
5539                 nft_clear(ctx->net, ext);
5540                 nft_setelem_data_activate(ctx->net, set, catchall->elem);
5541                 break;
5542         }
5543 }
5544
5545 static void nft_map_activate(const struct nft_ctx *ctx, struct nft_set *set)
5546 {
5547         struct nft_set_iter iter = {
5548                 .genmask        = nft_genmask_next(ctx->net),
5549                 .type           = NFT_ITER_UPDATE,
5550                 .fn             = nft_mapelem_activate,
5551         };
5552
5553         set->ops->walk(ctx, set, &iter);
5554         WARN_ON_ONCE(iter.err);
5555
5556         nft_map_catchall_activate(ctx, set);
5557 }
5558
5559 void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
5560 {
5561         if (nft_set_is_anonymous(set)) {
5562                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5563                         nft_map_activate(ctx, set);
5564
5565                 nft_clear(ctx->net, set);
5566         }
5567
5568         nft_use_inc_restore(&set->use);
5569 }
5570 EXPORT_SYMBOL_GPL(nf_tables_activate_set);
5571
5572 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
5573                               struct nft_set_binding *binding,
5574                               enum nft_trans_phase phase)
5575 {
5576         switch (phase) {
5577         case NFT_TRANS_PREPARE_ERROR:
5578                 nft_set_trans_unbind(ctx, set);
5579                 if (nft_set_is_anonymous(set))
5580                         nft_deactivate_next(ctx->net, set);
5581                 else
5582                         list_del_rcu(&binding->list);
5583
5584                 nft_use_dec(&set->use);
5585                 break;
5586         case NFT_TRANS_PREPARE:
5587                 if (nft_set_is_anonymous(set)) {
5588                         if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5589                                 nft_map_deactivate(ctx, set);
5590
5591                         nft_deactivate_next(ctx->net, set);
5592                 }
5593                 nft_use_dec(&set->use);
5594                 return;
5595         case NFT_TRANS_ABORT:
5596         case NFT_TRANS_RELEASE:
5597                 if (nft_set_is_anonymous(set) &&
5598                     set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
5599                         nft_map_deactivate(ctx, set);
5600
5601                 nft_use_dec(&set->use);
5602                 fallthrough;
5603         default:
5604                 nf_tables_unbind_set(ctx, set, binding,
5605                                      phase == NFT_TRANS_COMMIT);
5606         }
5607 }
5608 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
5609
5610 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
5611 {
5612         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
5613                 nft_set_destroy(ctx, set);
5614 }
5615 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
5616
5617 const struct nft_set_ext_type nft_set_ext_types[] = {
5618         [NFT_SET_EXT_KEY]               = {
5619                 .align  = __alignof__(u32),
5620         },
5621         [NFT_SET_EXT_DATA]              = {
5622                 .align  = __alignof__(u32),
5623         },
5624         [NFT_SET_EXT_EXPRESSIONS]       = {
5625                 .align  = __alignof__(struct nft_set_elem_expr),
5626         },
5627         [NFT_SET_EXT_OBJREF]            = {
5628                 .len    = sizeof(struct nft_object *),
5629                 .align  = __alignof__(struct nft_object *),
5630         },
5631         [NFT_SET_EXT_FLAGS]             = {
5632                 .len    = sizeof(u8),
5633                 .align  = __alignof__(u8),
5634         },
5635         [NFT_SET_EXT_TIMEOUT]           = {
5636                 .len    = sizeof(u64),
5637                 .align  = __alignof__(u64),
5638         },
5639         [NFT_SET_EXT_EXPIRATION]        = {
5640                 .len    = sizeof(u64),
5641                 .align  = __alignof__(u64),
5642         },
5643         [NFT_SET_EXT_USERDATA]          = {
5644                 .len    = sizeof(struct nft_userdata),
5645                 .align  = __alignof__(struct nft_userdata),
5646         },
5647         [NFT_SET_EXT_KEY_END]           = {
5648                 .align  = __alignof__(u32),
5649         },
5650 };
5651
5652 /*
5653  * Set elements
5654  */
5655
5656 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
5657         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
5658         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
5659         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
5660         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
5661         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
5662         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
5663                                             .len = NFT_USERDATA_MAXLEN },
5664         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
5665         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
5666                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
5667         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
5668         [NFTA_SET_ELEM_EXPRESSIONS]     = NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
5669 };
5670
5671 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
5672         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
5673                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
5674         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
5675                                             .len = NFT_SET_MAXNAMELEN - 1 },
5676         [NFTA_SET_ELEM_LIST_ELEMENTS]   = NLA_POLICY_NESTED_ARRAY(nft_set_elem_policy),
5677         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
5678 };
5679
5680 static int nft_set_elem_expr_dump(struct sk_buff *skb,
5681                                   const struct nft_set *set,
5682                                   const struct nft_set_ext *ext,
5683                                   bool reset)
5684 {
5685         struct nft_set_elem_expr *elem_expr;
5686         u32 size, num_exprs = 0;
5687         struct nft_expr *expr;
5688         struct nlattr *nest;
5689
5690         elem_expr = nft_set_ext_expr(ext);
5691         nft_setelem_expr_foreach(expr, elem_expr, size)
5692                 num_exprs++;
5693
5694         if (num_exprs == 1) {
5695                 expr = nft_setelem_expr_at(elem_expr, 0);
5696                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, reset) < 0)
5697                         return -1;
5698
5699                 return 0;
5700         } else if (num_exprs > 1) {
5701                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
5702                 if (nest == NULL)
5703                         goto nla_put_failure;
5704
5705                 nft_setelem_expr_foreach(expr, elem_expr, size) {
5706                         expr = nft_setelem_expr_at(elem_expr, size);
5707                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
5708                                 goto nla_put_failure;
5709                 }
5710                 nla_nest_end(skb, nest);
5711         }
5712         return 0;
5713
5714 nla_put_failure:
5715         return -1;
5716 }
5717
5718 static int nf_tables_fill_setelem(struct sk_buff *skb,
5719                                   const struct nft_set *set,
5720                                   const struct nft_elem_priv *elem_priv,
5721                                   bool reset)
5722 {
5723         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5724         unsigned char *b = skb_tail_pointer(skb);
5725         struct nlattr *nest;
5726
5727         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
5728         if (nest == NULL)
5729                 goto nla_put_failure;
5730
5731         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
5732             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
5733                           NFT_DATA_VALUE, set->klen) < 0)
5734                 goto nla_put_failure;
5735
5736         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
5737             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
5738                           NFT_DATA_VALUE, set->klen) < 0)
5739                 goto nla_put_failure;
5740
5741         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5742             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
5743                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
5744                           set->dlen) < 0)
5745                 goto nla_put_failure;
5746
5747         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
5748             nft_set_elem_expr_dump(skb, set, ext, reset))
5749                 goto nla_put_failure;
5750
5751         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5752             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
5753                            (*nft_set_ext_obj(ext))->key.name) < 0)
5754                 goto nla_put_failure;
5755
5756         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5757             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
5758                          htonl(*nft_set_ext_flags(ext))))
5759                 goto nla_put_failure;
5760
5761         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
5762             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
5763                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
5764                          NFTA_SET_ELEM_PAD))
5765                 goto nla_put_failure;
5766
5767         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5768                 u64 expires, now = get_jiffies_64();
5769
5770                 expires = *nft_set_ext_expiration(ext);
5771                 if (time_before64(now, expires))
5772                         expires -= now;
5773                 else
5774                         expires = 0;
5775
5776                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
5777                                  nf_jiffies64_to_msecs(expires),
5778                                  NFTA_SET_ELEM_PAD))
5779                         goto nla_put_failure;
5780         }
5781
5782         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
5783                 struct nft_userdata *udata;
5784
5785                 udata = nft_set_ext_userdata(ext);
5786                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
5787                             udata->len + 1, udata->data))
5788                         goto nla_put_failure;
5789         }
5790
5791         nla_nest_end(skb, nest);
5792         return 0;
5793
5794 nla_put_failure:
5795         nlmsg_trim(skb, b);
5796         return -EMSGSIZE;
5797 }
5798
5799 struct nft_set_dump_args {
5800         const struct netlink_callback   *cb;
5801         struct nft_set_iter             iter;
5802         struct sk_buff                  *skb;
5803         bool                            reset;
5804 };
5805
5806 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
5807                                   struct nft_set *set,
5808                                   const struct nft_set_iter *iter,
5809                                   struct nft_elem_priv *elem_priv)
5810 {
5811         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
5812         struct nft_set_dump_args *args;
5813
5814         if (!nft_set_elem_active(ext, iter->genmask))
5815                 return 0;
5816
5817         if (nft_set_elem_expired(ext) || nft_set_elem_is_dead(ext))
5818                 return 0;
5819
5820         args = container_of(iter, struct nft_set_dump_args, iter);
5821         return nf_tables_fill_setelem(args->skb, set, elem_priv, args->reset);
5822 }
5823
5824 static void audit_log_nft_set_reset(const struct nft_table *table,
5825                                     unsigned int base_seq,
5826                                     unsigned int nentries)
5827 {
5828         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
5829
5830         audit_log_nfcfg(buf, table->family, nentries,
5831                         AUDIT_NFT_OP_SETELEM_RESET, GFP_ATOMIC);
5832         kfree(buf);
5833 }
5834
5835 struct nft_set_dump_ctx {
5836         const struct nft_set    *set;
5837         struct nft_ctx          ctx;
5838         bool                    reset;
5839 };
5840
5841 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
5842                                  const struct nft_set *set, bool reset,
5843                                  unsigned int base_seq)
5844 {
5845         struct nft_set_elem_catchall *catchall;
5846         u8 genmask = nft_genmask_cur(net);
5847         struct nft_set_ext *ext;
5848         int ret = 0;
5849
5850         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5851                 ext = nft_set_elem_ext(set, catchall->elem);
5852                 if (!nft_set_elem_active(ext, genmask) ||
5853                     nft_set_elem_expired(ext))
5854                         continue;
5855
5856                 ret = nf_tables_fill_setelem(skb, set, catchall->elem, reset);
5857                 if (reset && !ret)
5858                         audit_log_nft_set_reset(set->table, base_seq, 1);
5859                 break;
5860         }
5861
5862         return ret;
5863 }
5864
5865 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
5866 {
5867         struct nft_set_dump_ctx *dump_ctx = cb->data;
5868         struct net *net = sock_net(skb->sk);
5869         struct nftables_pernet *nft_net;
5870         struct nft_table *table;
5871         struct nft_set *set;
5872         struct nft_set_dump_args args;
5873         bool set_found = false;
5874         struct nlmsghdr *nlh;
5875         struct nlattr *nest;
5876         u32 portid, seq;
5877         int event;
5878
5879         rcu_read_lock();
5880         nft_net = nft_pernet(net);
5881         cb->seq = READ_ONCE(nft_net->base_seq);
5882
5883         list_for_each_entry_rcu(table, &nft_net->tables, list) {
5884                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
5885                     dump_ctx->ctx.family != table->family)
5886                         continue;
5887
5888                 if (table != dump_ctx->ctx.table)
5889                         continue;
5890
5891                 list_for_each_entry_rcu(set, &table->sets, list) {
5892                         if (set == dump_ctx->set) {
5893                                 set_found = true;
5894                                 break;
5895                         }
5896                 }
5897                 break;
5898         }
5899
5900         if (!set_found) {
5901                 rcu_read_unlock();
5902                 return -ENOENT;
5903         }
5904
5905         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
5906         portid = NETLINK_CB(cb->skb).portid;
5907         seq    = cb->nlh->nlmsg_seq;
5908
5909         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
5910                            table->family, NFNETLINK_V0, nft_base_seq(net));
5911         if (!nlh)
5912                 goto nla_put_failure;
5913
5914         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
5915                 goto nla_put_failure;
5916         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
5917                 goto nla_put_failure;
5918
5919         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
5920         if (nest == NULL)
5921                 goto nla_put_failure;
5922
5923         args.cb                 = cb;
5924         args.skb                = skb;
5925         args.reset              = dump_ctx->reset;
5926         args.iter.genmask       = nft_genmask_cur(net);
5927         args.iter.type          = NFT_ITER_READ;
5928         args.iter.skip          = cb->args[0];
5929         args.iter.count         = 0;
5930         args.iter.err           = 0;
5931         args.iter.fn            = nf_tables_dump_setelem;
5932         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
5933
5934         if (!args.iter.err && args.iter.count == cb->args[0])
5935                 args.iter.err = nft_set_catchall_dump(net, skb, set,
5936                                                       dump_ctx->reset, cb->seq);
5937         nla_nest_end(skb, nest);
5938         nlmsg_end(skb, nlh);
5939
5940         rcu_read_unlock();
5941
5942         if (args.iter.err && args.iter.err != -EMSGSIZE)
5943                 return args.iter.err;
5944         if (args.iter.count == cb->args[0])
5945                 return 0;
5946
5947         cb->args[0] = args.iter.count;
5948         return skb->len;
5949
5950 nla_put_failure:
5951         rcu_read_unlock();
5952         return -ENOSPC;
5953 }
5954
5955 static int nf_tables_dumpreset_set(struct sk_buff *skb,
5956                                    struct netlink_callback *cb)
5957 {
5958         struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk));
5959         struct nft_set_dump_ctx *dump_ctx = cb->data;
5960         int ret, skip = cb->args[0];
5961
5962         mutex_lock(&nft_net->commit_mutex);
5963
5964         ret = nf_tables_dump_set(skb, cb);
5965
5966         if (cb->args[0] > skip)
5967                 audit_log_nft_set_reset(dump_ctx->ctx.table, cb->seq,
5968                                         cb->args[0] - skip);
5969
5970         mutex_unlock(&nft_net->commit_mutex);
5971
5972         return ret;
5973 }
5974
5975 static int nf_tables_dump_set_start(struct netlink_callback *cb)
5976 {
5977         struct nft_set_dump_ctx *dump_ctx = cb->data;
5978
5979         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
5980
5981         return cb->data ? 0 : -ENOMEM;
5982 }
5983
5984 static int nf_tables_dump_set_done(struct netlink_callback *cb)
5985 {
5986         kfree(cb->data);
5987         return 0;
5988 }
5989
5990 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
5991                                        const struct nft_ctx *ctx, u32 seq,
5992                                        u32 portid, int event, u16 flags,
5993                                        const struct nft_set *set,
5994                                        const struct nft_elem_priv *elem_priv,
5995                                        bool reset)
5996 {
5997         struct nlmsghdr *nlh;
5998         struct nlattr *nest;
5999         int err;
6000
6001         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
6002         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
6003                            NFNETLINK_V0, nft_base_seq(ctx->net));
6004         if (!nlh)
6005                 goto nla_put_failure;
6006
6007         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
6008                 goto nla_put_failure;
6009         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
6010                 goto nla_put_failure;
6011
6012         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
6013         if (nest == NULL)
6014                 goto nla_put_failure;
6015
6016         err = nf_tables_fill_setelem(skb, set, elem_priv, reset);
6017         if (err < 0)
6018                 goto nla_put_failure;
6019
6020         nla_nest_end(skb, nest);
6021
6022         nlmsg_end(skb, nlh);
6023         return 0;
6024
6025 nla_put_failure:
6026         nlmsg_trim(skb, nlh);
6027         return -1;
6028 }
6029
6030 static int nft_setelem_parse_flags(const struct nft_set *set,
6031                                    const struct nlattr *attr, u32 *flags)
6032 {
6033         if (attr == NULL)
6034                 return 0;
6035
6036         *flags = ntohl(nla_get_be32(attr));
6037         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
6038                 return -EOPNOTSUPP;
6039         if (!(set->flags & NFT_SET_INTERVAL) &&
6040             *flags & NFT_SET_ELEM_INTERVAL_END)
6041                 return -EINVAL;
6042         if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
6043             (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
6044                 return -EINVAL;
6045
6046         return 0;
6047 }
6048
6049 static int nft_setelem_parse_key(struct nft_ctx *ctx, const struct nft_set *set,
6050                                  struct nft_data *key, struct nlattr *attr)
6051 {
6052         struct nft_data_desc desc = {
6053                 .type   = NFT_DATA_VALUE,
6054                 .size   = NFT_DATA_VALUE_MAXLEN,
6055                 .len    = set->klen,
6056         };
6057
6058         return nft_data_init(ctx, key, &desc, attr);
6059 }
6060
6061 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
6062                                   struct nft_data_desc *desc,
6063                                   struct nft_data *data,
6064                                   struct nlattr *attr)
6065 {
6066         u32 dtype;
6067
6068         if (set->dtype == NFT_DATA_VERDICT)
6069                 dtype = NFT_DATA_VERDICT;
6070         else
6071                 dtype = NFT_DATA_VALUE;
6072
6073         desc->type = dtype;
6074         desc->size = NFT_DATA_VALUE_MAXLEN;
6075         desc->len = set->dlen;
6076         desc->flags = NFT_DATA_DESC_SETELEM;
6077
6078         return nft_data_init(ctx, data, desc, attr);
6079 }
6080
6081 static void *nft_setelem_catchall_get(const struct net *net,
6082                                       const struct nft_set *set)
6083 {
6084         struct nft_set_elem_catchall *catchall;
6085         u8 genmask = nft_genmask_cur(net);
6086         struct nft_set_ext *ext;
6087         void *priv = NULL;
6088
6089         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6090                 ext = nft_set_elem_ext(set, catchall->elem);
6091                 if (!nft_set_elem_active(ext, genmask) ||
6092                     nft_set_elem_expired(ext))
6093                         continue;
6094
6095                 priv = catchall->elem;
6096                 break;
6097         }
6098
6099         return priv;
6100 }
6101
6102 static int nft_setelem_get(struct nft_ctx *ctx, const struct nft_set *set,
6103                            struct nft_set_elem *elem, u32 flags)
6104 {
6105         void *priv;
6106
6107         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
6108                 priv = set->ops->get(ctx->net, set, elem, flags);
6109                 if (IS_ERR(priv))
6110                         return PTR_ERR(priv);
6111         } else {
6112                 priv = nft_setelem_catchall_get(ctx->net, set);
6113                 if (!priv)
6114                         return -ENOENT;
6115         }
6116         elem->priv = priv;
6117
6118         return 0;
6119 }
6120
6121 static int nft_get_set_elem(struct nft_ctx *ctx, const struct nft_set *set,
6122                             const struct nlattr *attr, bool reset)
6123 {
6124         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6125         struct nft_set_elem elem;
6126         struct sk_buff *skb;
6127         uint32_t flags = 0;
6128         int err;
6129
6130         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6131                                           nft_set_elem_policy, NULL);
6132         if (err < 0)
6133                 return err;
6134
6135         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6136         if (err < 0)
6137                 return err;
6138
6139         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6140                 return -EINVAL;
6141
6142         if (nla[NFTA_SET_ELEM_KEY]) {
6143                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6144                                             nla[NFTA_SET_ELEM_KEY]);
6145                 if (err < 0)
6146                         return err;
6147         }
6148
6149         if (nla[NFTA_SET_ELEM_KEY_END]) {
6150                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6151                                             nla[NFTA_SET_ELEM_KEY_END]);
6152                 if (err < 0)
6153                         return err;
6154         }
6155
6156         err = nft_setelem_get(ctx, set, &elem, flags);
6157         if (err < 0)
6158                 return err;
6159
6160         err = -ENOMEM;
6161         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
6162         if (skb == NULL)
6163                 return err;
6164
6165         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
6166                                           NFT_MSG_NEWSETELEM, 0, set, elem.priv,
6167                                           reset);
6168         if (err < 0)
6169                 goto err_fill_setelem;
6170
6171         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
6172
6173 err_fill_setelem:
6174         kfree_skb(skb);
6175         return err;
6176 }
6177
6178 static int nft_set_dump_ctx_init(struct nft_set_dump_ctx *dump_ctx,
6179                                  const struct sk_buff *skb,
6180                                  const struct nfnl_info *info,
6181                                  const struct nlattr * const nla[],
6182                                  bool reset)
6183 {
6184         struct netlink_ext_ack *extack = info->extack;
6185         u8 genmask = nft_genmask_cur(info->net);
6186         u8 family = info->nfmsg->nfgen_family;
6187         struct net *net = info->net;
6188         struct nft_table *table;
6189         struct nft_set *set;
6190
6191         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6192                                  genmask, 0);
6193         if (IS_ERR(table)) {
6194                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6195                 return PTR_ERR(table);
6196         }
6197
6198         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6199         if (IS_ERR(set)) {
6200                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
6201                 return PTR_ERR(set);
6202         }
6203
6204         nft_ctx_init(&dump_ctx->ctx, net, skb,
6205                      info->nlh, family, table, NULL, nla);
6206         dump_ctx->set = set;
6207         dump_ctx->reset = reset;
6208         return 0;
6209 }
6210
6211 /* called with rcu_read_lock held */
6212 static int nf_tables_getsetelem(struct sk_buff *skb,
6213                                 const struct nfnl_info *info,
6214                                 const struct nlattr * const nla[])
6215 {
6216         struct netlink_ext_ack *extack = info->extack;
6217         struct nft_set_dump_ctx dump_ctx;
6218         struct nlattr *attr;
6219         int rem, err = 0;
6220
6221         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6222                 struct netlink_dump_control c = {
6223                         .start = nf_tables_dump_set_start,
6224                         .dump = nf_tables_dump_set,
6225                         .done = nf_tables_dump_set_done,
6226                         .module = THIS_MODULE,
6227                 };
6228
6229                 err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false);
6230                 if (err)
6231                         return err;
6232
6233                 c.data = &dump_ctx;
6234                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6235         }
6236
6237         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6238                 return -EINVAL;
6239
6240         err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false);
6241         if (err)
6242                 return err;
6243
6244         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6245                 err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, false);
6246                 if (err < 0) {
6247                         NL_SET_BAD_ATTR(extack, attr);
6248                         break;
6249                 }
6250         }
6251
6252         return err;
6253 }
6254
6255 static int nf_tables_getsetelem_reset(struct sk_buff *skb,
6256                                       const struct nfnl_info *info,
6257                                       const struct nlattr * const nla[])
6258 {
6259         struct nftables_pernet *nft_net = nft_pernet(info->net);
6260         struct netlink_ext_ack *extack = info->extack;
6261         struct nft_set_dump_ctx dump_ctx;
6262         int rem, err = 0, nelems = 0;
6263         struct nlattr *attr;
6264
6265         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6266                 struct netlink_dump_control c = {
6267                         .start = nf_tables_dump_set_start,
6268                         .dump = nf_tables_dumpreset_set,
6269                         .done = nf_tables_dump_set_done,
6270                         .module = THIS_MODULE,
6271                 };
6272
6273                 err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true);
6274                 if (err)
6275                         return err;
6276
6277                 c.data = &dump_ctx;
6278                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6279         }
6280
6281         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6282                 return -EINVAL;
6283
6284         if (!try_module_get(THIS_MODULE))
6285                 return -EINVAL;
6286         rcu_read_unlock();
6287         mutex_lock(&nft_net->commit_mutex);
6288         rcu_read_lock();
6289
6290         err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true);
6291         if (err)
6292                 goto out_unlock;
6293
6294         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6295                 err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, true);
6296                 if (err < 0) {
6297                         NL_SET_BAD_ATTR(extack, attr);
6298                         break;
6299                 }
6300                 nelems++;
6301         }
6302         audit_log_nft_set_reset(dump_ctx.ctx.table, nft_net->base_seq, nelems);
6303
6304 out_unlock:
6305         rcu_read_unlock();
6306         mutex_unlock(&nft_net->commit_mutex);
6307         rcu_read_lock();
6308         module_put(THIS_MODULE);
6309
6310         return err;
6311 }
6312
6313 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
6314                                      const struct nft_set *set,
6315                                      const struct nft_elem_priv *elem_priv,
6316                                      int event)
6317 {
6318         struct nftables_pernet *nft_net;
6319         struct net *net = ctx->net;
6320         u32 portid = ctx->portid;
6321         struct sk_buff *skb;
6322         u16 flags = 0;
6323         int err;
6324
6325         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
6326                 return;
6327
6328         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
6329         if (skb == NULL)
6330                 goto err;
6331
6332         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
6333                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
6334
6335         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
6336                                           set, elem_priv, false);
6337         if (err < 0) {
6338                 kfree_skb(skb);
6339                 goto err;
6340         }
6341
6342         nft_net = nft_pernet(net);
6343         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
6344         return;
6345 err:
6346         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
6347 }
6348
6349 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
6350                                               int msg_type,
6351                                               struct nft_set *set)
6352 {
6353         struct nft_trans *trans;
6354
6355         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
6356         if (trans == NULL)
6357                 return NULL;
6358
6359         nft_trans_elem_set(trans) = set;
6360         return trans;
6361 }
6362
6363 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
6364                                          const struct nft_set *set,
6365                                          const struct nlattr *attr)
6366 {
6367         struct nft_expr *expr;
6368         int err;
6369
6370         expr = nft_expr_init(ctx, attr);
6371         if (IS_ERR(expr))
6372                 return expr;
6373
6374         err = -EOPNOTSUPP;
6375         if (expr->ops->type->flags & NFT_EXPR_GC) {
6376                 if (set->flags & NFT_SET_TIMEOUT)
6377                         goto err_set_elem_expr;
6378                 if (!set->ops->gc_init)
6379                         goto err_set_elem_expr;
6380                 set->ops->gc_init(set);
6381         }
6382
6383         return expr;
6384
6385 err_set_elem_expr:
6386         nft_expr_destroy(ctx, expr);
6387         return ERR_PTR(err);
6388 }
6389
6390 static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
6391 {
6392         len += nft_set_ext_types[id].len;
6393         if (len > tmpl->ext_len[id] ||
6394             len > U8_MAX)
6395                 return -1;
6396
6397         return 0;
6398 }
6399
6400 static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
6401                               void *to, const void *from, u32 len)
6402 {
6403         if (nft_set_ext_check(tmpl, id, len) < 0)
6404                 return -1;
6405
6406         memcpy(to, from, len);
6407
6408         return 0;
6409 }
6410
6411 struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set,
6412                                         const struct nft_set_ext_tmpl *tmpl,
6413                                         const u32 *key, const u32 *key_end,
6414                                         const u32 *data,
6415                                         u64 timeout, u64 expiration, gfp_t gfp)
6416 {
6417         struct nft_set_ext *ext;
6418         void *elem;
6419
6420         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
6421         if (elem == NULL)
6422                 return ERR_PTR(-ENOMEM);
6423
6424         ext = nft_set_elem_ext(set, elem);
6425         nft_set_ext_init(ext, tmpl);
6426
6427         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
6428             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
6429                                nft_set_ext_key(ext), key, set->klen) < 0)
6430                 goto err_ext_check;
6431
6432         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
6433             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
6434                                nft_set_ext_key_end(ext), key_end, set->klen) < 0)
6435                 goto err_ext_check;
6436
6437         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
6438             nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
6439                                nft_set_ext_data(ext), data, set->dlen) < 0)
6440                 goto err_ext_check;
6441
6442         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
6443                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
6444                 if (expiration == 0)
6445                         *nft_set_ext_expiration(ext) += timeout;
6446         }
6447         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
6448                 *nft_set_ext_timeout(ext) = timeout;
6449
6450         return elem;
6451
6452 err_ext_check:
6453         kfree(elem);
6454
6455         return ERR_PTR(-EINVAL);
6456 }
6457
6458 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6459                                         struct nft_expr *expr)
6460 {
6461         if (expr->ops->destroy_clone) {
6462                 expr->ops->destroy_clone(ctx, expr);
6463                 module_put(expr->ops->type->owner);
6464         } else {
6465                 nf_tables_expr_destroy(ctx, expr);
6466         }
6467 }
6468
6469 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
6470                                       struct nft_set_elem_expr *elem_expr)
6471 {
6472         struct nft_expr *expr;
6473         u32 size;
6474
6475         nft_setelem_expr_foreach(expr, elem_expr, size)
6476                 __nft_set_elem_expr_destroy(ctx, expr);
6477 }
6478
6479 /* Drop references and destroy. Called from gc, dynset and abort path. */
6480 void nft_set_elem_destroy(const struct nft_set *set,
6481                           const struct nft_elem_priv *elem_priv,
6482                           bool destroy_expr)
6483 {
6484         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6485         struct nft_ctx ctx = {
6486                 .net    = read_pnet(&set->net),
6487                 .family = set->table->family,
6488         };
6489
6490         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
6491         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6492                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6493         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6494                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
6495         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6496                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
6497
6498         kfree(elem_priv);
6499 }
6500 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
6501
6502 /* Destroy element. References have been already dropped in the preparation
6503  * path via nft_setelem_data_deactivate().
6504  */
6505 void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
6506                                 const struct nft_set *set,
6507                                 const struct nft_elem_priv *elem_priv)
6508 {
6509         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6510
6511         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
6512                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
6513
6514         kfree(elem_priv);
6515 }
6516
6517 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
6518                             struct nft_expr *expr_array[])
6519 {
6520         struct nft_expr *expr;
6521         int err, i, k;
6522
6523         for (i = 0; i < set->num_exprs; i++) {
6524                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
6525                 if (!expr)
6526                         goto err_expr;
6527
6528                 err = nft_expr_clone(expr, set->exprs[i]);
6529                 if (err < 0) {
6530                         kfree(expr);
6531                         goto err_expr;
6532                 }
6533                 expr_array[i] = expr;
6534         }
6535
6536         return 0;
6537
6538 err_expr:
6539         for (k = i - 1; k >= 0; k--)
6540                 nft_expr_destroy(ctx, expr_array[k]);
6541
6542         return -ENOMEM;
6543 }
6544
6545 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
6546                                    const struct nft_set_ext_tmpl *tmpl,
6547                                    const struct nft_set_ext *ext,
6548                                    struct nft_expr *expr_array[],
6549                                    u32 num_exprs)
6550 {
6551         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
6552         u32 len = sizeof(struct nft_set_elem_expr);
6553         struct nft_expr *expr;
6554         int i, err;
6555
6556         if (num_exprs == 0)
6557                 return 0;
6558
6559         for (i = 0; i < num_exprs; i++)
6560                 len += expr_array[i]->ops->size;
6561
6562         if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
6563                 return -EINVAL;
6564
6565         for (i = 0; i < num_exprs; i++) {
6566                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
6567                 err = nft_expr_clone(expr, expr_array[i]);
6568                 if (err < 0)
6569                         goto err_elem_expr_setup;
6570
6571                 elem_expr->size += expr_array[i]->ops->size;
6572                 nft_expr_destroy(ctx, expr_array[i]);
6573                 expr_array[i] = NULL;
6574         }
6575
6576         return 0;
6577
6578 err_elem_expr_setup:
6579         for (; i < num_exprs; i++) {
6580                 nft_expr_destroy(ctx, expr_array[i]);
6581                 expr_array[i] = NULL;
6582         }
6583
6584         return -ENOMEM;
6585 }
6586
6587 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
6588                                             const struct nft_set *set)
6589 {
6590         struct nft_set_elem_catchall *catchall;
6591         u8 genmask = nft_genmask_cur(net);
6592         struct nft_set_ext *ext;
6593
6594         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6595                 ext = nft_set_elem_ext(set, catchall->elem);
6596                 if (nft_set_elem_active(ext, genmask) &&
6597                     !nft_set_elem_expired(ext) &&
6598                     !nft_set_elem_is_dead(ext))
6599                         return ext;
6600         }
6601
6602         return NULL;
6603 }
6604 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6605
6606 static int nft_setelem_catchall_insert(const struct net *net,
6607                                        struct nft_set *set,
6608                                        const struct nft_set_elem *elem,
6609                                        struct nft_elem_priv **priv)
6610 {
6611         struct nft_set_elem_catchall *catchall;
6612         u8 genmask = nft_genmask_next(net);
6613         struct nft_set_ext *ext;
6614
6615         list_for_each_entry(catchall, &set->catchall_list, list) {
6616                 ext = nft_set_elem_ext(set, catchall->elem);
6617                 if (nft_set_elem_active(ext, genmask)) {
6618                         *priv = catchall->elem;
6619                         return -EEXIST;
6620                 }
6621         }
6622
6623         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
6624         if (!catchall)
6625                 return -ENOMEM;
6626
6627         catchall->elem = elem->priv;
6628         list_add_tail_rcu(&catchall->list, &set->catchall_list);
6629
6630         return 0;
6631 }
6632
6633 static int nft_setelem_insert(const struct net *net,
6634                               struct nft_set *set,
6635                               const struct nft_set_elem *elem,
6636                               struct nft_elem_priv **elem_priv,
6637                               unsigned int flags)
6638 {
6639         int ret;
6640
6641         if (flags & NFT_SET_ELEM_CATCHALL)
6642                 ret = nft_setelem_catchall_insert(net, set, elem, elem_priv);
6643         else
6644                 ret = set->ops->insert(net, set, elem, elem_priv);
6645
6646         return ret;
6647 }
6648
6649 static bool nft_setelem_is_catchall(const struct nft_set *set,
6650                                     const struct nft_elem_priv *elem_priv)
6651 {
6652         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6653
6654         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
6655             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
6656                 return true;
6657
6658         return false;
6659 }
6660
6661 static void nft_setelem_activate(struct net *net, struct nft_set *set,
6662                                  struct nft_elem_priv *elem_priv)
6663 {
6664         struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
6665
6666         if (nft_setelem_is_catchall(set, elem_priv)) {
6667                 nft_clear(net, ext);
6668         } else {
6669                 set->ops->activate(net, set, elem_priv);
6670         }
6671 }
6672
6673 static int nft_setelem_catchall_deactivate(const struct net *net,
6674                                            struct nft_set *set,
6675                                            struct nft_set_elem *elem)
6676 {
6677         struct nft_set_elem_catchall *catchall;
6678         struct nft_set_ext *ext;
6679
6680         list_for_each_entry(catchall, &set->catchall_list, list) {
6681                 ext = nft_set_elem_ext(set, catchall->elem);
6682                 if (!nft_is_active_next(net, ext))
6683                         continue;
6684
6685                 kfree(elem->priv);
6686                 elem->priv = catchall->elem;
6687                 nft_set_elem_change_active(net, set, ext);
6688                 return 0;
6689         }
6690
6691         return -ENOENT;
6692 }
6693
6694 static int __nft_setelem_deactivate(const struct net *net,
6695                                     struct nft_set *set,
6696                                     struct nft_set_elem *elem)
6697 {
6698         void *priv;
6699
6700         priv = set->ops->deactivate(net, set, elem);
6701         if (!priv)
6702                 return -ENOENT;
6703
6704         kfree(elem->priv);
6705         elem->priv = priv;
6706         set->ndeact++;
6707
6708         return 0;
6709 }
6710
6711 static int nft_setelem_deactivate(const struct net *net,
6712                                   struct nft_set *set,
6713                                   struct nft_set_elem *elem, u32 flags)
6714 {
6715         int ret;
6716
6717         if (flags & NFT_SET_ELEM_CATCHALL)
6718                 ret = nft_setelem_catchall_deactivate(net, set, elem);
6719         else
6720                 ret = __nft_setelem_deactivate(net, set, elem);
6721
6722         return ret;
6723 }
6724
6725 static void nft_setelem_catchall_destroy(struct nft_set_elem_catchall *catchall)
6726 {
6727         list_del_rcu(&catchall->list);
6728         kfree_rcu(catchall, rcu);
6729 }
6730
6731 static void nft_setelem_catchall_remove(const struct net *net,
6732                                         const struct nft_set *set,
6733                                         struct nft_elem_priv *elem_priv)
6734 {
6735         struct nft_set_elem_catchall *catchall, *next;
6736
6737         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6738                 if (catchall->elem == elem_priv) {
6739                         nft_setelem_catchall_destroy(catchall);
6740                         break;
6741                 }
6742         }
6743 }
6744
6745 static void nft_setelem_remove(const struct net *net,
6746                                const struct nft_set *set,
6747                                struct nft_elem_priv *elem_priv)
6748 {
6749         if (nft_setelem_is_catchall(set, elem_priv))
6750                 nft_setelem_catchall_remove(net, set, elem_priv);
6751         else
6752                 set->ops->remove(net, set, elem_priv);
6753 }
6754
6755 static bool nft_setelem_valid_key_end(const struct nft_set *set,
6756                                       struct nlattr **nla, u32 flags)
6757 {
6758         if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
6759                           (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
6760                 if (flags & NFT_SET_ELEM_INTERVAL_END)
6761                         return false;
6762
6763                 if (nla[NFTA_SET_ELEM_KEY_END] &&
6764                     flags & NFT_SET_ELEM_CATCHALL)
6765                         return false;
6766         } else {
6767                 if (nla[NFTA_SET_ELEM_KEY_END])
6768                         return false;
6769         }
6770
6771         return true;
6772 }
6773
6774 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
6775                             const struct nlattr *attr, u32 nlmsg_flags)
6776 {
6777         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
6778         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6779         u8 genmask = nft_genmask_next(ctx->net);
6780         u32 flags = 0, size = 0, num_exprs = 0;
6781         struct nft_set_ext_tmpl tmpl;
6782         struct nft_set_ext *ext, *ext2;
6783         struct nft_set_elem elem;
6784         struct nft_set_binding *binding;
6785         struct nft_elem_priv *elem_priv;
6786         struct nft_object *obj = NULL;
6787         struct nft_userdata *udata;
6788         struct nft_data_desc desc;
6789         enum nft_registers dreg;
6790         struct nft_trans *trans;
6791         u64 expiration;
6792         u64 timeout;
6793         int err, i;
6794         u8 ulen;
6795
6796         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6797                                           nft_set_elem_policy, NULL);
6798         if (err < 0)
6799                 return err;
6800
6801         nft_set_ext_prepare(&tmpl);
6802
6803         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6804         if (err < 0)
6805                 return err;
6806
6807         if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
6808             (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
6809                 return -EINVAL;
6810
6811         if (flags != 0) {
6812                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6813                 if (err < 0)
6814                         return err;
6815         }
6816
6817         if (set->flags & NFT_SET_MAP) {
6818                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
6819                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6820                         return -EINVAL;
6821         } else {
6822                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
6823                         return -EINVAL;
6824         }
6825
6826         if (set->flags & NFT_SET_OBJECT) {
6827                 if (!nla[NFTA_SET_ELEM_OBJREF] &&
6828                     !(flags & NFT_SET_ELEM_INTERVAL_END))
6829                         return -EINVAL;
6830         } else {
6831                 if (nla[NFTA_SET_ELEM_OBJREF])
6832                         return -EINVAL;
6833         }
6834
6835         if (!nft_setelem_valid_key_end(set, nla, flags))
6836                 return -EINVAL;
6837
6838         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
6839              (nla[NFTA_SET_ELEM_DATA] ||
6840               nla[NFTA_SET_ELEM_OBJREF] ||
6841               nla[NFTA_SET_ELEM_TIMEOUT] ||
6842               nla[NFTA_SET_ELEM_EXPIRATION] ||
6843               nla[NFTA_SET_ELEM_USERDATA] ||
6844               nla[NFTA_SET_ELEM_EXPR] ||
6845               nla[NFTA_SET_ELEM_KEY_END] ||
6846               nla[NFTA_SET_ELEM_EXPRESSIONS]))
6847                 return -EINVAL;
6848
6849         timeout = 0;
6850         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
6851                 if (!(set->flags & NFT_SET_TIMEOUT))
6852                         return -EINVAL;
6853                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
6854                                             &timeout);
6855                 if (err)
6856                         return err;
6857         } else if (set->flags & NFT_SET_TIMEOUT &&
6858                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6859                 timeout = READ_ONCE(set->timeout);
6860         }
6861
6862         expiration = 0;
6863         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
6864                 if (!(set->flags & NFT_SET_TIMEOUT))
6865                         return -EINVAL;
6866                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
6867                                             &expiration);
6868                 if (err)
6869                         return err;
6870         }
6871
6872         if (nla[NFTA_SET_ELEM_EXPR]) {
6873                 struct nft_expr *expr;
6874
6875                 if (set->num_exprs && set->num_exprs != 1)
6876                         return -EOPNOTSUPP;
6877
6878                 expr = nft_set_elem_expr_alloc(ctx, set,
6879                                                nla[NFTA_SET_ELEM_EXPR]);
6880                 if (IS_ERR(expr))
6881                         return PTR_ERR(expr);
6882
6883                 expr_array[0] = expr;
6884                 num_exprs = 1;
6885
6886                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
6887                         err = -EOPNOTSUPP;
6888                         goto err_set_elem_expr;
6889                 }
6890         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
6891                 struct nft_expr *expr;
6892                 struct nlattr *tmp;
6893                 int left;
6894
6895                 i = 0;
6896                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
6897                         if (i == NFT_SET_EXPR_MAX ||
6898                             (set->num_exprs && set->num_exprs == i)) {
6899                                 err = -E2BIG;
6900                                 goto err_set_elem_expr;
6901                         }
6902                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
6903                                 err = -EINVAL;
6904                                 goto err_set_elem_expr;
6905                         }
6906                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
6907                         if (IS_ERR(expr)) {
6908                                 err = PTR_ERR(expr);
6909                                 goto err_set_elem_expr;
6910                         }
6911                         expr_array[i] = expr;
6912                         num_exprs++;
6913
6914                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
6915                                 err = -EOPNOTSUPP;
6916                                 goto err_set_elem_expr;
6917                         }
6918                         i++;
6919                 }
6920                 if (set->num_exprs && set->num_exprs != i) {
6921                         err = -EOPNOTSUPP;
6922                         goto err_set_elem_expr;
6923                 }
6924         } else if (set->num_exprs > 0 &&
6925                    !(flags & NFT_SET_ELEM_INTERVAL_END)) {
6926                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
6927                 if (err < 0)
6928                         goto err_set_elem_expr_clone;
6929
6930                 num_exprs = set->num_exprs;
6931         }
6932
6933         if (nla[NFTA_SET_ELEM_KEY]) {
6934                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6935                                             nla[NFTA_SET_ELEM_KEY]);
6936                 if (err < 0)
6937                         goto err_set_elem_expr;
6938
6939                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6940                 if (err < 0)
6941                         goto err_parse_key;
6942         }
6943
6944         if (nla[NFTA_SET_ELEM_KEY_END]) {
6945                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6946                                             nla[NFTA_SET_ELEM_KEY_END]);
6947                 if (err < 0)
6948                         goto err_parse_key;
6949
6950                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6951                 if (err < 0)
6952                         goto err_parse_key_end;
6953         }
6954
6955         if (timeout > 0) {
6956                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
6957                 if (err < 0)
6958                         goto err_parse_key_end;
6959
6960                 if (timeout != READ_ONCE(set->timeout)) {
6961                         err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
6962                         if (err < 0)
6963                                 goto err_parse_key_end;
6964                 }
6965         }
6966
6967         if (num_exprs) {
6968                 for (i = 0; i < num_exprs; i++)
6969                         size += expr_array[i]->ops->size;
6970
6971                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
6972                                              sizeof(struct nft_set_elem_expr) + size);
6973                 if (err < 0)
6974                         goto err_parse_key_end;
6975         }
6976
6977         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
6978                 obj = nft_obj_lookup(ctx->net, ctx->table,
6979                                      nla[NFTA_SET_ELEM_OBJREF],
6980                                      set->objtype, genmask);
6981                 if (IS_ERR(obj)) {
6982                         err = PTR_ERR(obj);
6983                         obj = NULL;
6984                         goto err_parse_key_end;
6985                 }
6986
6987                 if (!nft_use_inc(&obj->use)) {
6988                         err = -EMFILE;
6989                         obj = NULL;
6990                         goto err_parse_key_end;
6991                 }
6992
6993                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
6994                 if (err < 0)
6995                         goto err_parse_key_end;
6996         }
6997
6998         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
6999                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
7000                                              nla[NFTA_SET_ELEM_DATA]);
7001                 if (err < 0)
7002                         goto err_parse_key_end;
7003
7004                 dreg = nft_type_to_reg(set->dtype);
7005                 list_for_each_entry(binding, &set->bindings, list) {
7006                         struct nft_ctx bind_ctx = {
7007                                 .net    = ctx->net,
7008                                 .family = ctx->family,
7009                                 .table  = ctx->table,
7010                                 .chain  = (struct nft_chain *)binding->chain,
7011                         };
7012
7013                         if (!(binding->flags & NFT_SET_MAP))
7014                                 continue;
7015
7016                         err = nft_validate_register_store(&bind_ctx, dreg,
7017                                                           &elem.data.val,
7018                                                           desc.type, desc.len);
7019                         if (err < 0)
7020                                 goto err_parse_data;
7021
7022                         if (desc.type == NFT_DATA_VERDICT &&
7023                             (elem.data.val.verdict.code == NFT_GOTO ||
7024                              elem.data.val.verdict.code == NFT_JUMP))
7025                                 nft_validate_state_update(ctx->table,
7026                                                           NFT_VALIDATE_NEED);
7027                 }
7028
7029                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
7030                 if (err < 0)
7031                         goto err_parse_data;
7032         }
7033
7034         /* The full maximum length of userdata can exceed the maximum
7035          * offset value (U8_MAX) for following extensions, therefor it
7036          * must be the last extension added.
7037          */
7038         ulen = 0;
7039         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
7040                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
7041                 if (ulen > 0) {
7042                         err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
7043                                                      ulen);
7044                         if (err < 0)
7045                                 goto err_parse_data;
7046                 }
7047         }
7048
7049         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7050                                       elem.key_end.val.data, elem.data.val.data,
7051                                       timeout, expiration, GFP_KERNEL_ACCOUNT);
7052         if (IS_ERR(elem.priv)) {
7053                 err = PTR_ERR(elem.priv);
7054                 goto err_parse_data;
7055         }
7056
7057         ext = nft_set_elem_ext(set, elem.priv);
7058         if (flags)
7059                 *nft_set_ext_flags(ext) = flags;
7060
7061         if (obj)
7062                 *nft_set_ext_obj(ext) = obj;
7063
7064         if (ulen > 0) {
7065                 if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
7066                         err = -EINVAL;
7067                         goto err_elem_free;
7068                 }
7069                 udata = nft_set_ext_userdata(ext);
7070                 udata->len = ulen - 1;
7071                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
7072         }
7073         err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
7074         if (err < 0)
7075                 goto err_elem_free;
7076
7077         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
7078         if (trans == NULL) {
7079                 err = -ENOMEM;
7080                 goto err_elem_free;
7081         }
7082
7083         ext->genmask = nft_genmask_cur(ctx->net);
7084
7085         err = nft_setelem_insert(ctx->net, set, &elem, &elem_priv, flags);
7086         if (err) {
7087                 if (err == -EEXIST) {
7088                         ext2 = nft_set_elem_ext(set, elem_priv);
7089                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
7090                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
7091                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
7092                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
7093                                 goto err_element_clash;
7094                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
7095                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
7096                              memcmp(nft_set_ext_data(ext),
7097                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
7098                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
7099                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
7100                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
7101                                 goto err_element_clash;
7102                         else if (!(nlmsg_flags & NLM_F_EXCL))
7103                                 err = 0;
7104                 } else if (err == -ENOTEMPTY) {
7105                         /* ENOTEMPTY reports overlapping between this element
7106                          * and an existing one.
7107                          */
7108                         err = -EEXIST;
7109                 }
7110                 goto err_element_clash;
7111         }
7112
7113         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
7114                 unsigned int max = set->size ? set->size + set->ndeact : UINT_MAX;
7115
7116                 if (!atomic_add_unless(&set->nelems, 1, max)) {
7117                         err = -ENFILE;
7118                         goto err_set_full;
7119                 }
7120         }
7121
7122         nft_trans_elem_priv(trans) = elem.priv;
7123         nft_trans_commit_list_add_tail(ctx->net, trans);
7124         return 0;
7125
7126 err_set_full:
7127         nft_setelem_remove(ctx->net, set, elem.priv);
7128 err_element_clash:
7129         kfree(trans);
7130 err_elem_free:
7131         nf_tables_set_elem_destroy(ctx, set, elem.priv);
7132 err_parse_data:
7133         if (nla[NFTA_SET_ELEM_DATA] != NULL)
7134                 nft_data_release(&elem.data.val, desc.type);
7135 err_parse_key_end:
7136         if (obj)
7137                 nft_use_dec_restore(&obj->use);
7138
7139         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7140 err_parse_key:
7141         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7142 err_set_elem_expr:
7143         for (i = 0; i < num_exprs && expr_array[i]; i++)
7144                 nft_expr_destroy(ctx, expr_array[i]);
7145 err_set_elem_expr_clone:
7146         return err;
7147 }
7148
7149 static int nf_tables_newsetelem(struct sk_buff *skb,
7150                                 const struct nfnl_info *info,
7151                                 const struct nlattr * const nla[])
7152 {
7153         struct netlink_ext_ack *extack = info->extack;
7154         u8 genmask = nft_genmask_next(info->net);
7155         u8 family = info->nfmsg->nfgen_family;
7156         struct net *net = info->net;
7157         const struct nlattr *attr;
7158         struct nft_table *table;
7159         struct nft_set *set;
7160         struct nft_ctx ctx;
7161         int rem, err;
7162
7163         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
7164                 return -EINVAL;
7165
7166         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7167                                  genmask, NETLINK_CB(skb).portid);
7168         if (IS_ERR(table)) {
7169                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7170                 return PTR_ERR(table);
7171         }
7172
7173         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
7174                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
7175         if (IS_ERR(set)) {
7176                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
7177                 return PTR_ERR(set);
7178         }
7179
7180         if (!list_empty(&set->bindings) &&
7181             (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
7182                 return -EBUSY;
7183
7184         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7185
7186         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7187                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
7188                 if (err < 0) {
7189                         NL_SET_BAD_ATTR(extack, attr);
7190                         return err;
7191                 }
7192         }
7193
7194         if (table->validate_state == NFT_VALIDATE_DO)
7195                 return nft_table_validate(net, table);
7196
7197         return 0;
7198 }
7199
7200 /**
7201  *      nft_data_hold - hold a nft_data item
7202  *
7203  *      @data: struct nft_data to release
7204  *      @type: type of data
7205  *
7206  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
7207  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
7208  *      NFT_GOTO verdicts. This function must be called on active data objects
7209  *      from the second phase of the commit protocol.
7210  */
7211 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
7212 {
7213         struct nft_chain *chain;
7214
7215         if (type == NFT_DATA_VERDICT) {
7216                 switch (data->verdict.code) {
7217                 case NFT_JUMP:
7218                 case NFT_GOTO:
7219                         chain = data->verdict.chain;
7220                         nft_use_inc_restore(&chain->use);
7221                         break;
7222                 }
7223         }
7224 }
7225
7226 static void nft_setelem_data_activate(const struct net *net,
7227                                       const struct nft_set *set,
7228                                       struct nft_elem_priv *elem_priv)
7229 {
7230         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7231
7232         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7233                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
7234         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7235                 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
7236 }
7237
7238 void nft_setelem_data_deactivate(const struct net *net,
7239                                  const struct nft_set *set,
7240                                  struct nft_elem_priv *elem_priv)
7241 {
7242         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7243
7244         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
7245                 nft_data_release(nft_set_ext_data(ext), set->dtype);
7246         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
7247                 nft_use_dec(&(*nft_set_ext_obj(ext))->use);
7248 }
7249
7250 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
7251                            const struct nlattr *attr)
7252 {
7253         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
7254         struct nft_set_ext_tmpl tmpl;
7255         struct nft_set_elem elem;
7256         struct nft_set_ext *ext;
7257         struct nft_trans *trans;
7258         u32 flags = 0;
7259         int err;
7260
7261         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
7262                                           nft_set_elem_policy, NULL);
7263         if (err < 0)
7264                 return err;
7265
7266         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
7267         if (err < 0)
7268                 return err;
7269
7270         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
7271                 return -EINVAL;
7272
7273         if (!nft_setelem_valid_key_end(set, nla, flags))
7274                 return -EINVAL;
7275
7276         nft_set_ext_prepare(&tmpl);
7277
7278         if (flags != 0) {
7279                 err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
7280                 if (err < 0)
7281                         return err;
7282         }
7283
7284         if (nla[NFTA_SET_ELEM_KEY]) {
7285                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
7286                                             nla[NFTA_SET_ELEM_KEY]);
7287                 if (err < 0)
7288                         return err;
7289
7290                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
7291                 if (err < 0)
7292                         goto fail_elem;
7293         }
7294
7295         if (nla[NFTA_SET_ELEM_KEY_END]) {
7296                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
7297                                             nla[NFTA_SET_ELEM_KEY_END]);
7298                 if (err < 0)
7299                         goto fail_elem;
7300
7301                 err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
7302                 if (err < 0)
7303                         goto fail_elem_key_end;
7304         }
7305
7306         err = -ENOMEM;
7307         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
7308                                       elem.key_end.val.data, NULL, 0, 0,
7309                                       GFP_KERNEL_ACCOUNT);
7310         if (IS_ERR(elem.priv)) {
7311                 err = PTR_ERR(elem.priv);
7312                 goto fail_elem_key_end;
7313         }
7314
7315         ext = nft_set_elem_ext(set, elem.priv);
7316         if (flags)
7317                 *nft_set_ext_flags(ext) = flags;
7318
7319         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
7320         if (trans == NULL)
7321                 goto fail_trans;
7322
7323         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
7324         if (err < 0)
7325                 goto fail_ops;
7326
7327         nft_setelem_data_deactivate(ctx->net, set, elem.priv);
7328
7329         nft_trans_elem_priv(trans) = elem.priv;
7330         nft_trans_commit_list_add_tail(ctx->net, trans);
7331         return 0;
7332
7333 fail_ops:
7334         kfree(trans);
7335 fail_trans:
7336         kfree(elem.priv);
7337 fail_elem_key_end:
7338         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
7339 fail_elem:
7340         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
7341         return err;
7342 }
7343
7344 static int nft_setelem_flush(const struct nft_ctx *ctx,
7345                              struct nft_set *set,
7346                              const struct nft_set_iter *iter,
7347                              struct nft_elem_priv *elem_priv)
7348 {
7349         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
7350         struct nft_trans *trans;
7351
7352         if (!nft_set_elem_active(ext, iter->genmask))
7353                 return 0;
7354
7355         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7356                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
7357         if (!trans)
7358                 return -ENOMEM;
7359
7360         set->ops->flush(ctx->net, set, elem_priv);
7361         set->ndeact++;
7362
7363         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
7364         nft_trans_elem_set(trans) = set;
7365         nft_trans_elem_priv(trans) = elem_priv;
7366         nft_trans_commit_list_add_tail(ctx->net, trans);
7367
7368         return 0;
7369 }
7370
7371 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
7372                                     struct nft_set *set,
7373                                     struct nft_elem_priv *elem_priv)
7374 {
7375         struct nft_trans *trans;
7376
7377         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
7378                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
7379         if (!trans)
7380                 return -ENOMEM;
7381
7382         nft_setelem_data_deactivate(ctx->net, set, elem_priv);
7383         nft_trans_elem_set(trans) = set;
7384         nft_trans_elem_priv(trans) = elem_priv;
7385         nft_trans_commit_list_add_tail(ctx->net, trans);
7386
7387         return 0;
7388 }
7389
7390 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
7391                                   struct nft_set *set)
7392 {
7393         u8 genmask = nft_genmask_next(ctx->net);
7394         struct nft_set_elem_catchall *catchall;
7395         struct nft_set_ext *ext;
7396         int ret = 0;
7397
7398         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
7399                 ext = nft_set_elem_ext(set, catchall->elem);
7400                 if (!nft_set_elem_active(ext, genmask))
7401                         continue;
7402
7403                 ret = __nft_set_catchall_flush(ctx, set, catchall->elem);
7404                 if (ret < 0)
7405                         break;
7406                 nft_set_elem_change_active(ctx->net, set, ext);
7407         }
7408
7409         return ret;
7410 }
7411
7412 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
7413 {
7414         struct nft_set_iter iter = {
7415                 .genmask        = genmask,
7416                 .type           = NFT_ITER_UPDATE,
7417                 .fn             = nft_setelem_flush,
7418         };
7419
7420         set->ops->walk(ctx, set, &iter);
7421         if (!iter.err)
7422                 iter.err = nft_set_catchall_flush(ctx, set);
7423
7424         return iter.err;
7425 }
7426
7427 static int nf_tables_delsetelem(struct sk_buff *skb,
7428                                 const struct nfnl_info *info,
7429                                 const struct nlattr * const nla[])
7430 {
7431         struct netlink_ext_ack *extack = info->extack;
7432         u8 genmask = nft_genmask_next(info->net);
7433         u8 family = info->nfmsg->nfgen_family;
7434         struct net *net = info->net;
7435         const struct nlattr *attr;
7436         struct nft_table *table;
7437         struct nft_set *set;
7438         struct nft_ctx ctx;
7439         int rem, err = 0;
7440
7441         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
7442                                  genmask, NETLINK_CB(skb).portid);
7443         if (IS_ERR(table)) {
7444                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
7445                 return PTR_ERR(table);
7446         }
7447
7448         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
7449         if (IS_ERR(set)) {
7450                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
7451                 return PTR_ERR(set);
7452         }
7453
7454         if (nft_set_is_anonymous(set))
7455                 return -EOPNOTSUPP;
7456
7457         if (!list_empty(&set->bindings) && (set->flags & NFT_SET_CONSTANT))
7458                 return -EBUSY;
7459
7460         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7461
7462         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
7463                 return nft_set_flush(&ctx, set, genmask);
7464
7465         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
7466                 err = nft_del_setelem(&ctx, set, attr);
7467                 if (err == -ENOENT &&
7468                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
7469                         continue;
7470
7471                 if (err < 0) {
7472                         NL_SET_BAD_ATTR(extack, attr);
7473                         return err;
7474                 }
7475         }
7476
7477         return 0;
7478 }
7479
7480 /*
7481  * Stateful objects
7482  */
7483
7484 /**
7485  *      nft_register_obj- register nf_tables stateful object type
7486  *      @obj_type: object type
7487  *
7488  *      Registers the object type for use with nf_tables. Returns zero on
7489  *      success or a negative errno code otherwise.
7490  */
7491 int nft_register_obj(struct nft_object_type *obj_type)
7492 {
7493         if (obj_type->type == NFT_OBJECT_UNSPEC)
7494                 return -EINVAL;
7495
7496         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7497         list_add_rcu(&obj_type->list, &nf_tables_objects);
7498         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7499         return 0;
7500 }
7501 EXPORT_SYMBOL_GPL(nft_register_obj);
7502
7503 /**
7504  *      nft_unregister_obj - unregister nf_tables object type
7505  *      @obj_type: object type
7506  *
7507  *      Unregisters the object type for use with nf_tables.
7508  */
7509 void nft_unregister_obj(struct nft_object_type *obj_type)
7510 {
7511         nfnl_lock(NFNL_SUBSYS_NFTABLES);
7512         list_del_rcu(&obj_type->list);
7513         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
7514 }
7515 EXPORT_SYMBOL_GPL(nft_unregister_obj);
7516
7517 struct nft_object *nft_obj_lookup(const struct net *net,
7518                                   const struct nft_table *table,
7519                                   const struct nlattr *nla, u32 objtype,
7520                                   u8 genmask)
7521 {
7522         struct nft_object_hash_key k = { .table = table };
7523         char search[NFT_OBJ_MAXNAMELEN];
7524         struct rhlist_head *tmp, *list;
7525         struct nft_object *obj;
7526
7527         nla_strscpy(search, nla, sizeof(search));
7528         k.name = search;
7529
7530         WARN_ON_ONCE(!rcu_read_lock_held() &&
7531                      !lockdep_commit_lock_is_held(net));
7532
7533         rcu_read_lock();
7534         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
7535         if (!list)
7536                 goto out;
7537
7538         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
7539                 if (objtype == obj->ops->type->type &&
7540                     nft_active_genmask(obj, genmask)) {
7541                         rcu_read_unlock();
7542                         return obj;
7543                 }
7544         }
7545 out:
7546         rcu_read_unlock();
7547         return ERR_PTR(-ENOENT);
7548 }
7549 EXPORT_SYMBOL_GPL(nft_obj_lookup);
7550
7551 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
7552                                                   const struct nlattr *nla,
7553                                                   u32 objtype, u8 genmask)
7554 {
7555         struct nft_object *obj;
7556
7557         list_for_each_entry(obj, &table->objects, list) {
7558                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
7559                     objtype == obj->ops->type->type &&
7560                     nft_active_genmask(obj, genmask))
7561                         return obj;
7562         }
7563         return ERR_PTR(-ENOENT);
7564 }
7565
7566 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
7567         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
7568                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
7569         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
7570                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
7571         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
7572         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
7573         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
7574         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
7575                                     .len = NFT_USERDATA_MAXLEN },
7576 };
7577
7578 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
7579                                        const struct nft_object_type *type,
7580                                        const struct nlattr *attr)
7581 {
7582         struct nlattr **tb;
7583         const struct nft_object_ops *ops;
7584         struct nft_object *obj;
7585         int err = -ENOMEM;
7586
7587         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
7588         if (!tb)
7589                 goto err1;
7590
7591         if (attr) {
7592                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
7593                                                   type->policy, NULL);
7594                 if (err < 0)
7595                         goto err2;
7596         } else {
7597                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
7598         }
7599
7600         if (type->select_ops) {
7601                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
7602                 if (IS_ERR(ops)) {
7603                         err = PTR_ERR(ops);
7604                         goto err2;
7605                 }
7606         } else {
7607                 ops = type->ops;
7608         }
7609
7610         err = -ENOMEM;
7611         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
7612         if (!obj)
7613                 goto err2;
7614
7615         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
7616         if (err < 0)
7617                 goto err3;
7618
7619         obj->ops = ops;
7620
7621         kfree(tb);
7622         return obj;
7623 err3:
7624         kfree(obj);
7625 err2:
7626         kfree(tb);
7627 err1:
7628         return ERR_PTR(err);
7629 }
7630
7631 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
7632                            struct nft_object *obj, bool reset)
7633 {
7634         struct nlattr *nest;
7635
7636         nest = nla_nest_start_noflag(skb, attr);
7637         if (!nest)
7638                 goto nla_put_failure;
7639         if (obj->ops->dump(skb, obj, reset) < 0)
7640                 goto nla_put_failure;
7641         nla_nest_end(skb, nest);
7642         return 0;
7643
7644 nla_put_failure:
7645         return -1;
7646 }
7647
7648 static const struct nft_object_type *__nft_obj_type_get(u32 objtype, u8 family)
7649 {
7650         const struct nft_object_type *type;
7651
7652         list_for_each_entry_rcu(type, &nf_tables_objects, list) {
7653                 if (type->family != NFPROTO_UNSPEC &&
7654                     type->family != family)
7655                         continue;
7656
7657                 if (objtype == type->type)
7658                         return type;
7659         }
7660         return NULL;
7661 }
7662
7663 static const struct nft_object_type *
7664 nft_obj_type_get(struct net *net, u32 objtype, u8 family)
7665 {
7666         const struct nft_object_type *type;
7667
7668         rcu_read_lock();
7669         type = __nft_obj_type_get(objtype, family);
7670         if (type != NULL && try_module_get(type->owner)) {
7671                 rcu_read_unlock();
7672                 return type;
7673         }
7674         rcu_read_unlock();
7675
7676         lockdep_nfnl_nft_mutex_not_held();
7677 #ifdef CONFIG_MODULES
7678         if (type == NULL) {
7679                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
7680                         return ERR_PTR(-EAGAIN);
7681         }
7682 #endif
7683         return ERR_PTR(-ENOENT);
7684 }
7685
7686 static int nf_tables_updobj(const struct nft_ctx *ctx,
7687                             const struct nft_object_type *type,
7688                             const struct nlattr *attr,
7689                             struct nft_object *obj)
7690 {
7691         struct nft_object *newobj;
7692         struct nft_trans *trans;
7693         int err = -ENOMEM;
7694
7695         if (!try_module_get(type->owner))
7696                 return -ENOENT;
7697
7698         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
7699                                 sizeof(struct nft_trans_obj));
7700         if (!trans)
7701                 goto err_trans;
7702
7703         newobj = nft_obj_init(ctx, type, attr);
7704         if (IS_ERR(newobj)) {
7705                 err = PTR_ERR(newobj);
7706                 goto err_free_trans;
7707         }
7708
7709         nft_trans_obj(trans) = obj;
7710         nft_trans_obj_update(trans) = true;
7711         nft_trans_obj_newobj(trans) = newobj;
7712         nft_trans_commit_list_add_tail(ctx->net, trans);
7713
7714         return 0;
7715
7716 err_free_trans:
7717         kfree(trans);
7718 err_trans:
7719         module_put(type->owner);
7720         return err;
7721 }
7722
7723 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
7724                             const struct nlattr * const nla[])
7725 {
7726         struct netlink_ext_ack *extack = info->extack;
7727         u8 genmask = nft_genmask_next(info->net);
7728         u8 family = info->nfmsg->nfgen_family;
7729         const struct nft_object_type *type;
7730         struct net *net = info->net;
7731         struct nft_table *table;
7732         struct nft_object *obj;
7733         struct nft_ctx ctx;
7734         u32 objtype;
7735         int err;
7736
7737         if (!nla[NFTA_OBJ_TYPE] ||
7738             !nla[NFTA_OBJ_NAME] ||
7739             !nla[NFTA_OBJ_DATA])
7740                 return -EINVAL;
7741
7742         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
7743                                  NETLINK_CB(skb).portid);
7744         if (IS_ERR(table)) {
7745                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
7746                 return PTR_ERR(table);
7747         }
7748
7749         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7750         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
7751         if (IS_ERR(obj)) {
7752                 err = PTR_ERR(obj);
7753                 if (err != -ENOENT) {
7754                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7755                         return err;
7756                 }
7757         } else {
7758                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7759                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
7760                         return -EEXIST;
7761                 }
7762                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
7763                         return -EOPNOTSUPP;
7764
7765                 type = __nft_obj_type_get(objtype, family);
7766                 if (WARN_ON_ONCE(!type))
7767                         return -ENOENT;
7768
7769                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7770
7771                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
7772         }
7773
7774         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7775
7776         if (!nft_use_inc(&table->use))
7777                 return -EMFILE;
7778
7779         type = nft_obj_type_get(net, objtype, family);
7780         if (IS_ERR(type)) {
7781                 err = PTR_ERR(type);
7782                 goto err_type;
7783         }
7784
7785         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
7786         if (IS_ERR(obj)) {
7787                 err = PTR_ERR(obj);
7788                 goto err_init;
7789         }
7790         obj->key.table = table;
7791         obj->handle = nf_tables_alloc_handle(table);
7792
7793         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
7794         if (!obj->key.name) {
7795                 err = -ENOMEM;
7796                 goto err_strdup;
7797         }
7798
7799         if (nla[NFTA_OBJ_USERDATA]) {
7800                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
7801                 if (obj->udata == NULL)
7802                         goto err_userdata;
7803
7804                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
7805         }
7806
7807         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
7808         if (err < 0)
7809                 goto err_trans;
7810
7811         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
7812                               nft_objname_ht_params);
7813         if (err < 0)
7814                 goto err_obj_ht;
7815
7816         list_add_tail_rcu(&obj->list, &table->objects);
7817
7818         return 0;
7819 err_obj_ht:
7820         /* queued in transaction log */
7821         INIT_LIST_HEAD(&obj->list);
7822         return err;
7823 err_trans:
7824         kfree(obj->udata);
7825 err_userdata:
7826         kfree(obj->key.name);
7827 err_strdup:
7828         if (obj->ops->destroy)
7829                 obj->ops->destroy(&ctx, obj);
7830         kfree(obj);
7831 err_init:
7832         module_put(type->owner);
7833 err_type:
7834         nft_use_dec_restore(&table->use);
7835
7836         return err;
7837 }
7838
7839 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
7840                                    u32 portid, u32 seq, int event, u32 flags,
7841                                    int family, const struct nft_table *table,
7842                                    struct nft_object *obj, bool reset)
7843 {
7844         struct nlmsghdr *nlh;
7845
7846         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7847         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7848                            NFNETLINK_V0, nft_base_seq(net));
7849         if (!nlh)
7850                 goto nla_put_failure;
7851
7852         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
7853             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
7854             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
7855                          NFTA_OBJ_PAD))
7856                 goto nla_put_failure;
7857
7858         if (event == NFT_MSG_DELOBJ) {
7859                 nlmsg_end(skb, nlh);
7860                 return 0;
7861         }
7862
7863         if (nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
7864             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
7865             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
7866                 goto nla_put_failure;
7867
7868         if (obj->udata &&
7869             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
7870                 goto nla_put_failure;
7871
7872         nlmsg_end(skb, nlh);
7873         return 0;
7874
7875 nla_put_failure:
7876         nlmsg_trim(skb, nlh);
7877         return -1;
7878 }
7879
7880 static void audit_log_obj_reset(const struct nft_table *table,
7881                                 unsigned int base_seq, unsigned int nentries)
7882 {
7883         char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
7884
7885         audit_log_nfcfg(buf, table->family, nentries,
7886                         AUDIT_NFT_OP_OBJ_RESET, GFP_ATOMIC);
7887         kfree(buf);
7888 }
7889
7890 struct nft_obj_dump_ctx {
7891         unsigned int    s_idx;
7892         char            *table;
7893         u32             type;
7894         bool            reset;
7895 };
7896
7897 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
7898 {
7899         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7900         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7901         struct net *net = sock_net(skb->sk);
7902         int family = nfmsg->nfgen_family;
7903         struct nftables_pernet *nft_net;
7904         const struct nft_table *table;
7905         unsigned int entries = 0;
7906         struct nft_object *obj;
7907         unsigned int idx = 0;
7908         int rc = 0;
7909
7910         rcu_read_lock();
7911         nft_net = nft_pernet(net);
7912         cb->seq = READ_ONCE(nft_net->base_seq);
7913
7914         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7915                 if (family != NFPROTO_UNSPEC && family != table->family)
7916                         continue;
7917
7918                 entries = 0;
7919                 list_for_each_entry_rcu(obj, &table->objects, list) {
7920                         if (!nft_is_active(net, obj))
7921                                 goto cont;
7922                         if (idx < ctx->s_idx)
7923                                 goto cont;
7924                         if (ctx->table && strcmp(ctx->table, table->name))
7925                                 goto cont;
7926                         if (ctx->type != NFT_OBJECT_UNSPEC &&
7927                             obj->ops->type->type != ctx->type)
7928                                 goto cont;
7929
7930                         rc = nf_tables_fill_obj_info(skb, net,
7931                                                      NETLINK_CB(cb->skb).portid,
7932                                                      cb->nlh->nlmsg_seq,
7933                                                      NFT_MSG_NEWOBJ,
7934                                                      NLM_F_MULTI | NLM_F_APPEND,
7935                                                      table->family, table,
7936                                                      obj, ctx->reset);
7937                         if (rc < 0)
7938                                 break;
7939
7940                         entries++;
7941                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7942 cont:
7943                         idx++;
7944                 }
7945                 if (ctx->reset && entries)
7946                         audit_log_obj_reset(table, nft_net->base_seq, entries);
7947                 if (rc < 0)
7948                         break;
7949         }
7950         rcu_read_unlock();
7951
7952         ctx->s_idx = idx;
7953         return skb->len;
7954 }
7955
7956 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
7957 {
7958         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7959         const struct nlattr * const *nla = cb->data;
7960
7961         BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
7962
7963         if (nla[NFTA_OBJ_TABLE]) {
7964                 ctx->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
7965                 if (!ctx->table)
7966                         return -ENOMEM;
7967         }
7968
7969         if (nla[NFTA_OBJ_TYPE])
7970                 ctx->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
7971
7972         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
7973                 ctx->reset = true;
7974
7975         return 0;
7976 }
7977
7978 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
7979 {
7980         struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
7981
7982         kfree(ctx->table);
7983
7984         return 0;
7985 }
7986
7987 /* called with rcu_read_lock held */
7988 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
7989                             const struct nlattr * const nla[])
7990 {
7991         struct netlink_ext_ack *extack = info->extack;
7992         u8 genmask = nft_genmask_cur(info->net);
7993         u8 family = info->nfmsg->nfgen_family;
7994         const struct nft_table *table;
7995         struct net *net = info->net;
7996         struct nft_object *obj;
7997         struct sk_buff *skb2;
7998         bool reset = false;
7999         u32 objtype;
8000         int err;
8001
8002         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8003                 struct netlink_dump_control c = {
8004                         .start = nf_tables_dump_obj_start,
8005                         .dump = nf_tables_dump_obj,
8006                         .done = nf_tables_dump_obj_done,
8007                         .module = THIS_MODULE,
8008                         .data = (void *)nla,
8009                 };
8010
8011                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8012         }
8013
8014         if (!nla[NFTA_OBJ_NAME] ||
8015             !nla[NFTA_OBJ_TYPE])
8016                 return -EINVAL;
8017
8018         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
8019         if (IS_ERR(table)) {
8020                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
8021                 return PTR_ERR(table);
8022         }
8023
8024         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
8025         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
8026         if (IS_ERR(obj)) {
8027                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
8028                 return PTR_ERR(obj);
8029         }
8030
8031         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8032         if (!skb2)
8033                 return -ENOMEM;
8034
8035         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
8036                 reset = true;
8037
8038         if (reset) {
8039                 const struct nftables_pernet *nft_net;
8040                 char *buf;
8041
8042                 nft_net = nft_pernet(net);
8043                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
8044
8045                 audit_log_nfcfg(buf,
8046                                 family,
8047                                 1,
8048                                 AUDIT_NFT_OP_OBJ_RESET,
8049                                 GFP_ATOMIC);
8050                 kfree(buf);
8051         }
8052
8053         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
8054                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
8055                                       family, table, obj, reset);
8056         if (err < 0)
8057                 goto err_fill_obj_info;
8058
8059         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
8060
8061 err_fill_obj_info:
8062         kfree_skb(skb2);
8063         return err;
8064 }
8065
8066 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
8067 {
8068         if (obj->ops->destroy)
8069                 obj->ops->destroy(ctx, obj);
8070
8071         module_put(obj->ops->type->owner);
8072         kfree(obj->key.name);
8073         kfree(obj->udata);
8074         kfree(obj);
8075 }
8076
8077 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
8078                             const struct nlattr * const nla[])
8079 {
8080         struct netlink_ext_ack *extack = info->extack;
8081         u8 genmask = nft_genmask_next(info->net);
8082         u8 family = info->nfmsg->nfgen_family;
8083         struct net *net = info->net;
8084         const struct nlattr *attr;
8085         struct nft_table *table;
8086         struct nft_object *obj;
8087         struct nft_ctx ctx;
8088         u32 objtype;
8089
8090         if (!nla[NFTA_OBJ_TYPE] ||
8091             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
8092                 return -EINVAL;
8093
8094         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
8095                                  NETLINK_CB(skb).portid);
8096         if (IS_ERR(table)) {
8097                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
8098                 return PTR_ERR(table);
8099         }
8100
8101         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
8102         if (nla[NFTA_OBJ_HANDLE]) {
8103                 attr = nla[NFTA_OBJ_HANDLE];
8104                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
8105         } else {
8106                 attr = nla[NFTA_OBJ_NAME];
8107                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
8108         }
8109
8110         if (IS_ERR(obj)) {
8111                 if (PTR_ERR(obj) == -ENOENT &&
8112                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
8113                         return 0;
8114
8115                 NL_SET_BAD_ATTR(extack, attr);
8116                 return PTR_ERR(obj);
8117         }
8118         if (obj->use > 0) {
8119                 NL_SET_BAD_ATTR(extack, attr);
8120                 return -EBUSY;
8121         }
8122
8123         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8124
8125         return nft_delobj(&ctx, obj);
8126 }
8127
8128 static void
8129 __nft_obj_notify(struct net *net, const struct nft_table *table,
8130                  struct nft_object *obj, u32 portid, u32 seq, int event,
8131                  u16 flags, int family, int report, gfp_t gfp)
8132 {
8133         struct nftables_pernet *nft_net = nft_pernet(net);
8134         struct sk_buff *skb;
8135         int err;
8136
8137         if (!report &&
8138             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
8139                 return;
8140
8141         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
8142         if (skb == NULL)
8143                 goto err;
8144
8145         err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
8146                                       flags & (NLM_F_CREATE | NLM_F_EXCL),
8147                                       family, table, obj, false);
8148         if (err < 0) {
8149                 kfree_skb(skb);
8150                 goto err;
8151         }
8152
8153         nft_notify_enqueue(skb, report, &nft_net->notify_list);
8154         return;
8155 err:
8156         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
8157 }
8158
8159 void nft_obj_notify(struct net *net, const struct nft_table *table,
8160                     struct nft_object *obj, u32 portid, u32 seq, int event,
8161                     u16 flags, int family, int report, gfp_t gfp)
8162 {
8163         struct nftables_pernet *nft_net = nft_pernet(net);
8164         char *buf = kasprintf(gfp, "%s:%u",
8165                               table->name, nft_net->base_seq);
8166
8167         audit_log_nfcfg(buf,
8168                         family,
8169                         obj->handle,
8170                         event == NFT_MSG_NEWOBJ ?
8171                                  AUDIT_NFT_OP_OBJ_REGISTER :
8172                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
8173                         gfp);
8174         kfree(buf);
8175
8176         __nft_obj_notify(net, table, obj, portid, seq, event,
8177                          flags, family, report, gfp);
8178 }
8179 EXPORT_SYMBOL_GPL(nft_obj_notify);
8180
8181 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
8182                                  struct nft_object *obj, int event)
8183 {
8184         __nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid,
8185                          ctx->seq, event, ctx->flags, ctx->family,
8186                          ctx->report, GFP_KERNEL);
8187 }
8188
8189 /*
8190  * Flow tables
8191  */
8192 void nft_register_flowtable_type(struct nf_flowtable_type *type)
8193 {
8194         nfnl_lock(NFNL_SUBSYS_NFTABLES);
8195         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
8196         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
8197 }
8198 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
8199
8200 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
8201 {
8202         nfnl_lock(NFNL_SUBSYS_NFTABLES);
8203         list_del_rcu(&type->list);
8204         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
8205 }
8206 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
8207
8208 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
8209         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
8210                                             .len = NFT_NAME_MAXLEN - 1 },
8211         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
8212                                             .len = NFT_NAME_MAXLEN - 1 },
8213         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
8214         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
8215         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
8216 };
8217
8218 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
8219                                            const struct nlattr *nla, u8 genmask)
8220 {
8221         struct nft_flowtable *flowtable;
8222
8223         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8224                 if (!nla_strcmp(nla, flowtable->name) &&
8225                     nft_active_genmask(flowtable, genmask))
8226                         return flowtable;
8227         }
8228         return ERR_PTR(-ENOENT);
8229 }
8230 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
8231
8232 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
8233                                     struct nft_flowtable *flowtable,
8234                                     enum nft_trans_phase phase)
8235 {
8236         switch (phase) {
8237         case NFT_TRANS_PREPARE_ERROR:
8238         case NFT_TRANS_PREPARE:
8239         case NFT_TRANS_ABORT:
8240         case NFT_TRANS_RELEASE:
8241                 nft_use_dec(&flowtable->use);
8242                 fallthrough;
8243         default:
8244                 return;
8245         }
8246 }
8247 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
8248
8249 static struct nft_flowtable *
8250 nft_flowtable_lookup_byhandle(const struct nft_table *table,
8251                               const struct nlattr *nla, u8 genmask)
8252 {
8253        struct nft_flowtable *flowtable;
8254
8255        list_for_each_entry(flowtable, &table->flowtables, list) {
8256                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
8257                    nft_active_genmask(flowtable, genmask))
8258                        return flowtable;
8259        }
8260        return ERR_PTR(-ENOENT);
8261 }
8262
8263 struct nft_flowtable_hook {
8264         u32                     num;
8265         int                     priority;
8266         struct list_head        list;
8267 };
8268
8269 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
8270         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
8271         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
8272         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
8273 };
8274
8275 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
8276                                     const struct nlattr * const nla[],
8277                                     struct nft_flowtable_hook *flowtable_hook,
8278                                     struct nft_flowtable *flowtable,
8279                                     struct netlink_ext_ack *extack, bool add)
8280 {
8281         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
8282         struct nft_hook *hook;
8283         int hooknum, priority;
8284         int err;
8285
8286         INIT_LIST_HEAD(&flowtable_hook->list);
8287
8288         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
8289                                           nla[NFTA_FLOWTABLE_HOOK],
8290                                           nft_flowtable_hook_policy, NULL);
8291         if (err < 0)
8292                 return err;
8293
8294         if (add) {
8295                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
8296                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8297                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8298                         return -ENOENT;
8299                 }
8300
8301                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8302                 if (hooknum != NF_NETDEV_INGRESS)
8303                         return -EOPNOTSUPP;
8304
8305                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8306
8307                 flowtable_hook->priority        = priority;
8308                 flowtable_hook->num             = hooknum;
8309         } else {
8310                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
8311                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
8312                         if (hooknum != flowtable->hooknum)
8313                                 return -EOPNOTSUPP;
8314                 }
8315
8316                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
8317                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
8318                         if (priority != flowtable->data.priority)
8319                                 return -EOPNOTSUPP;
8320                 }
8321
8322                 flowtable_hook->priority        = flowtable->data.priority;
8323                 flowtable_hook->num             = flowtable->hooknum;
8324         }
8325
8326         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
8327                 err = nf_tables_parse_netdev_hooks(ctx->net,
8328                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
8329                                                    &flowtable_hook->list,
8330                                                    extack);
8331                 if (err < 0)
8332                         return err;
8333         }
8334
8335         list_for_each_entry(hook, &flowtable_hook->list, list) {
8336                 hook->ops.pf            = NFPROTO_NETDEV;
8337                 hook->ops.hooknum       = flowtable_hook->num;
8338                 hook->ops.priority      = flowtable_hook->priority;
8339                 hook->ops.priv          = &flowtable->data;
8340                 hook->ops.hook          = flowtable->data.type->hook;
8341         }
8342
8343         return err;
8344 }
8345
8346 /* call under rcu_read_lock */
8347 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
8348 {
8349         const struct nf_flowtable_type *type;
8350
8351         list_for_each_entry_rcu(type, &nf_tables_flowtables, list) {
8352                 if (family == type->family)
8353                         return type;
8354         }
8355         return NULL;
8356 }
8357
8358 static const struct nf_flowtable_type *
8359 nft_flowtable_type_get(struct net *net, u8 family)
8360 {
8361         const struct nf_flowtable_type *type;
8362
8363         rcu_read_lock();
8364         type = __nft_flowtable_type_get(family);
8365         if (type != NULL && try_module_get(type->owner)) {
8366                 rcu_read_unlock();
8367                 return type;
8368         }
8369         rcu_read_unlock();
8370
8371         lockdep_nfnl_nft_mutex_not_held();
8372 #ifdef CONFIG_MODULES
8373         if (type == NULL) {
8374                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
8375                         return ERR_PTR(-EAGAIN);
8376         }
8377 #endif
8378         return ERR_PTR(-ENOENT);
8379 }
8380
8381 /* Only called from error and netdev event paths. */
8382 static void nft_unregister_flowtable_hook(struct net *net,
8383                                           struct nft_flowtable *flowtable,
8384                                           struct nft_hook *hook)
8385 {
8386         nf_unregister_net_hook(net, &hook->ops);
8387         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
8388                                     FLOW_BLOCK_UNBIND);
8389 }
8390
8391 static void __nft_unregister_flowtable_net_hooks(struct net *net,
8392                                                  struct list_head *hook_list,
8393                                                  bool release_netdev)
8394 {
8395         struct nft_hook *hook, *next;
8396
8397         list_for_each_entry_safe(hook, next, hook_list, list) {
8398                 nf_unregister_net_hook(net, &hook->ops);
8399                 if (release_netdev) {
8400                         list_del(&hook->list);
8401                         kfree_rcu(hook, rcu);
8402                 }
8403         }
8404 }
8405
8406 static void nft_unregister_flowtable_net_hooks(struct net *net,
8407                                                struct list_head *hook_list)
8408 {
8409         __nft_unregister_flowtable_net_hooks(net, hook_list, false);
8410 }
8411
8412 static int nft_register_flowtable_net_hooks(struct net *net,
8413                                             struct nft_table *table,
8414                                             struct list_head *hook_list,
8415                                             struct nft_flowtable *flowtable)
8416 {
8417         struct nft_hook *hook, *hook2, *next;
8418         struct nft_flowtable *ft;
8419         int err, i = 0;
8420
8421         list_for_each_entry(hook, hook_list, list) {
8422                 list_for_each_entry(ft, &table->flowtables, list) {
8423                         if (!nft_is_active_next(net, ft))
8424                                 continue;
8425
8426                         list_for_each_entry(hook2, &ft->hook_list, list) {
8427                                 if (hook->ops.dev == hook2->ops.dev &&
8428                                     hook->ops.pf == hook2->ops.pf) {
8429                                         err = -EEXIST;
8430                                         goto err_unregister_net_hooks;
8431                                 }
8432                         }
8433                 }
8434
8435                 err = flowtable->data.type->setup(&flowtable->data,
8436                                                   hook->ops.dev,
8437                                                   FLOW_BLOCK_BIND);
8438                 if (err < 0)
8439                         goto err_unregister_net_hooks;
8440
8441                 err = nf_register_net_hook(net, &hook->ops);
8442                 if (err < 0) {
8443                         flowtable->data.type->setup(&flowtable->data,
8444                                                     hook->ops.dev,
8445                                                     FLOW_BLOCK_UNBIND);
8446                         goto err_unregister_net_hooks;
8447                 }
8448
8449                 i++;
8450         }
8451
8452         return 0;
8453
8454 err_unregister_net_hooks:
8455         list_for_each_entry_safe(hook, next, hook_list, list) {
8456                 if (i-- <= 0)
8457                         break;
8458
8459                 nft_unregister_flowtable_hook(net, flowtable, hook);
8460                 list_del_rcu(&hook->list);
8461                 kfree_rcu(hook, rcu);
8462         }
8463
8464         return err;
8465 }
8466
8467 static void nft_hooks_destroy(struct list_head *hook_list)
8468 {
8469         struct nft_hook *hook, *next;
8470
8471         list_for_each_entry_safe(hook, next, hook_list, list) {
8472                 list_del_rcu(&hook->list);
8473                 kfree_rcu(hook, rcu);
8474         }
8475 }
8476
8477 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
8478                                 struct nft_flowtable *flowtable,
8479                                 struct netlink_ext_ack *extack)
8480 {
8481         const struct nlattr * const *nla = ctx->nla;
8482         struct nft_flowtable_hook flowtable_hook;
8483         struct nft_hook *hook, *next;
8484         struct nft_trans *trans;
8485         bool unregister = false;
8486         u32 flags;
8487         int err;
8488
8489         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8490                                        extack, false);
8491         if (err < 0)
8492                 return err;
8493
8494         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8495                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
8496                         list_del(&hook->list);
8497                         kfree(hook);
8498                 }
8499         }
8500
8501         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8502                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8503                 if (flags & ~NFT_FLOWTABLE_MASK) {
8504                         err = -EOPNOTSUPP;
8505                         goto err_flowtable_update_hook;
8506                 }
8507                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
8508                     (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
8509                         err = -EOPNOTSUPP;
8510                         goto err_flowtable_update_hook;
8511                 }
8512         } else {
8513                 flags = flowtable->data.flags;
8514         }
8515
8516         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
8517                                                &flowtable_hook.list, flowtable);
8518         if (err < 0)
8519                 goto err_flowtable_update_hook;
8520
8521         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
8522                                 sizeof(struct nft_trans_flowtable));
8523         if (!trans) {
8524                 unregister = true;
8525                 err = -ENOMEM;
8526                 goto err_flowtable_update_hook;
8527         }
8528
8529         nft_trans_flowtable_flags(trans) = flags;
8530         nft_trans_flowtable(trans) = flowtable;
8531         nft_trans_flowtable_update(trans) = true;
8532         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8533         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
8534
8535         nft_trans_commit_list_add_tail(ctx->net, trans);
8536
8537         return 0;
8538
8539 err_flowtable_update_hook:
8540         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
8541                 if (unregister)
8542                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
8543                 list_del_rcu(&hook->list);
8544                 kfree_rcu(hook, rcu);
8545         }
8546
8547         return err;
8548
8549 }
8550
8551 static int nf_tables_newflowtable(struct sk_buff *skb,
8552                                   const struct nfnl_info *info,
8553                                   const struct nlattr * const nla[])
8554 {
8555         struct netlink_ext_ack *extack = info->extack;
8556         struct nft_flowtable_hook flowtable_hook;
8557         u8 genmask = nft_genmask_next(info->net);
8558         u8 family = info->nfmsg->nfgen_family;
8559         const struct nf_flowtable_type *type;
8560         struct nft_flowtable *flowtable;
8561         struct net *net = info->net;
8562         struct nft_table *table;
8563         struct nft_trans *trans;
8564         struct nft_ctx ctx;
8565         int err;
8566
8567         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8568             !nla[NFTA_FLOWTABLE_NAME] ||
8569             !nla[NFTA_FLOWTABLE_HOOK])
8570                 return -EINVAL;
8571
8572         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8573                                  genmask, NETLINK_CB(skb).portid);
8574         if (IS_ERR(table)) {
8575                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8576                 return PTR_ERR(table);
8577         }
8578
8579         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8580                                          genmask);
8581         if (IS_ERR(flowtable)) {
8582                 err = PTR_ERR(flowtable);
8583                 if (err != -ENOENT) {
8584                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8585                         return err;
8586                 }
8587         } else {
8588                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
8589                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8590                         return -EEXIST;
8591                 }
8592
8593                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8594
8595                 return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
8596         }
8597
8598         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8599
8600         if (!nft_use_inc(&table->use))
8601                 return -EMFILE;
8602
8603         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
8604         if (!flowtable) {
8605                 err = -ENOMEM;
8606                 goto flowtable_alloc;
8607         }
8608
8609         flowtable->table = table;
8610         flowtable->handle = nf_tables_alloc_handle(table);
8611         INIT_LIST_HEAD(&flowtable->hook_list);
8612
8613         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
8614         if (!flowtable->name) {
8615                 err = -ENOMEM;
8616                 goto err1;
8617         }
8618
8619         type = nft_flowtable_type_get(net, family);
8620         if (IS_ERR(type)) {
8621                 err = PTR_ERR(type);
8622                 goto err2;
8623         }
8624
8625         if (nla[NFTA_FLOWTABLE_FLAGS]) {
8626                 flowtable->data.flags =
8627                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
8628                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
8629                         err = -EOPNOTSUPP;
8630                         goto err3;
8631                 }
8632         }
8633
8634         write_pnet(&flowtable->data.net, net);
8635         flowtable->data.type = type;
8636         err = type->init(&flowtable->data);
8637         if (err < 0)
8638                 goto err3;
8639
8640         err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
8641                                        extack, true);
8642         if (err < 0)
8643                 goto err_flowtable_parse_hooks;
8644
8645         list_splice(&flowtable_hook.list, &flowtable->hook_list);
8646         flowtable->data.priority = flowtable_hook.priority;
8647         flowtable->hooknum = flowtable_hook.num;
8648
8649         trans = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
8650         if (IS_ERR(trans)) {
8651                 err = PTR_ERR(trans);
8652                 goto err_flowtable_trans;
8653         }
8654
8655         /* This must be LAST to ensure no packets are walking over this flowtable. */
8656         err = nft_register_flowtable_net_hooks(ctx.net, table,
8657                                                &flowtable->hook_list,
8658                                                flowtable);
8659         if (err < 0)
8660                 goto err_flowtable_hooks;
8661
8662         list_add_tail_rcu(&flowtable->list, &table->flowtables);
8663
8664         return 0;
8665
8666 err_flowtable_hooks:
8667         nft_trans_destroy(trans);
8668 err_flowtable_trans:
8669         nft_hooks_destroy(&flowtable->hook_list);
8670 err_flowtable_parse_hooks:
8671         flowtable->data.type->free(&flowtable->data);
8672 err3:
8673         module_put(type->owner);
8674 err2:
8675         kfree(flowtable->name);
8676 err1:
8677         kfree(flowtable);
8678 flowtable_alloc:
8679         nft_use_dec_restore(&table->use);
8680
8681         return err;
8682 }
8683
8684 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
8685 {
8686         struct nft_hook *this, *next;
8687
8688         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
8689                 list_del(&this->list);
8690                 kfree(this);
8691         }
8692 }
8693
8694 static int nft_delflowtable_hook(struct nft_ctx *ctx,
8695                                  struct nft_flowtable *flowtable,
8696                                  struct netlink_ext_ack *extack)
8697 {
8698         const struct nlattr * const *nla = ctx->nla;
8699         struct nft_flowtable_hook flowtable_hook;
8700         LIST_HEAD(flowtable_del_list);
8701         struct nft_hook *this, *hook;
8702         struct nft_trans *trans;
8703         int err;
8704
8705         err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
8706                                        extack, false);
8707         if (err < 0)
8708                 return err;
8709
8710         list_for_each_entry(this, &flowtable_hook.list, list) {
8711                 hook = nft_hook_list_find(&flowtable->hook_list, this);
8712                 if (!hook) {
8713                         err = -ENOENT;
8714                         goto err_flowtable_del_hook;
8715                 }
8716                 list_move(&hook->list, &flowtable_del_list);
8717         }
8718
8719         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
8720                                 sizeof(struct nft_trans_flowtable));
8721         if (!trans) {
8722                 err = -ENOMEM;
8723                 goto err_flowtable_del_hook;
8724         }
8725
8726         nft_trans_flowtable(trans) = flowtable;
8727         nft_trans_flowtable_update(trans) = true;
8728         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
8729         list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
8730         nft_flowtable_hook_release(&flowtable_hook);
8731
8732         nft_trans_commit_list_add_tail(ctx->net, trans);
8733
8734         return 0;
8735
8736 err_flowtable_del_hook:
8737         list_splice(&flowtable_del_list, &flowtable->hook_list);
8738         nft_flowtable_hook_release(&flowtable_hook);
8739
8740         return err;
8741 }
8742
8743 static int nf_tables_delflowtable(struct sk_buff *skb,
8744                                   const struct nfnl_info *info,
8745                                   const struct nlattr * const nla[])
8746 {
8747         struct netlink_ext_ack *extack = info->extack;
8748         u8 genmask = nft_genmask_next(info->net);
8749         u8 family = info->nfmsg->nfgen_family;
8750         struct nft_flowtable *flowtable;
8751         struct net *net = info->net;
8752         const struct nlattr *attr;
8753         struct nft_table *table;
8754         struct nft_ctx ctx;
8755
8756         if (!nla[NFTA_FLOWTABLE_TABLE] ||
8757             (!nla[NFTA_FLOWTABLE_NAME] &&
8758              !nla[NFTA_FLOWTABLE_HANDLE]))
8759                 return -EINVAL;
8760
8761         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8762                                  genmask, NETLINK_CB(skb).portid);
8763         if (IS_ERR(table)) {
8764                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8765                 return PTR_ERR(table);
8766         }
8767
8768         if (nla[NFTA_FLOWTABLE_HANDLE]) {
8769                 attr = nla[NFTA_FLOWTABLE_HANDLE];
8770                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
8771         } else {
8772                 attr = nla[NFTA_FLOWTABLE_NAME];
8773                 flowtable = nft_flowtable_lookup(table, attr, genmask);
8774         }
8775
8776         if (IS_ERR(flowtable)) {
8777                 if (PTR_ERR(flowtable) == -ENOENT &&
8778                     NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
8779                         return 0;
8780
8781                 NL_SET_BAD_ATTR(extack, attr);
8782                 return PTR_ERR(flowtable);
8783         }
8784
8785         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
8786
8787         if (nla[NFTA_FLOWTABLE_HOOK])
8788                 return nft_delflowtable_hook(&ctx, flowtable, extack);
8789
8790         if (flowtable->use > 0) {
8791                 NL_SET_BAD_ATTR(extack, attr);
8792                 return -EBUSY;
8793         }
8794
8795         return nft_delflowtable(&ctx, flowtable);
8796 }
8797
8798 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
8799                                          u32 portid, u32 seq, int event,
8800                                          u32 flags, int family,
8801                                          struct nft_flowtable *flowtable,
8802                                          struct list_head *hook_list)
8803 {
8804         struct nlattr *nest, *nest_devs;
8805         struct nft_hook *hook;
8806         struct nlmsghdr *nlh;
8807
8808         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
8809         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
8810                            NFNETLINK_V0, nft_base_seq(net));
8811         if (!nlh)
8812                 goto nla_put_failure;
8813
8814         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
8815             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
8816             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
8817                          NFTA_FLOWTABLE_PAD))
8818                 goto nla_put_failure;
8819
8820         if (event == NFT_MSG_DELFLOWTABLE && !hook_list) {
8821                 nlmsg_end(skb, nlh);
8822                 return 0;
8823         }
8824
8825         if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
8826             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
8827                 goto nla_put_failure;
8828
8829         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
8830         if (!nest)
8831                 goto nla_put_failure;
8832         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
8833             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
8834                 goto nla_put_failure;
8835
8836         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
8837         if (!nest_devs)
8838                 goto nla_put_failure;
8839
8840         if (!hook_list)
8841                 hook_list = &flowtable->hook_list;
8842
8843         list_for_each_entry_rcu(hook, hook_list, list) {
8844                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
8845                         goto nla_put_failure;
8846         }
8847         nla_nest_end(skb, nest_devs);
8848         nla_nest_end(skb, nest);
8849
8850         nlmsg_end(skb, nlh);
8851         return 0;
8852
8853 nla_put_failure:
8854         nlmsg_trim(skb, nlh);
8855         return -1;
8856 }
8857
8858 struct nft_flowtable_filter {
8859         char            *table;
8860 };
8861
8862 static int nf_tables_dump_flowtable(struct sk_buff *skb,
8863                                     struct netlink_callback *cb)
8864 {
8865         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
8866         struct nft_flowtable_filter *filter = cb->data;
8867         unsigned int idx = 0, s_idx = cb->args[0];
8868         struct net *net = sock_net(skb->sk);
8869         int family = nfmsg->nfgen_family;
8870         struct nft_flowtable *flowtable;
8871         struct nftables_pernet *nft_net;
8872         const struct nft_table *table;
8873
8874         rcu_read_lock();
8875         nft_net = nft_pernet(net);
8876         cb->seq = READ_ONCE(nft_net->base_seq);
8877
8878         list_for_each_entry_rcu(table, &nft_net->tables, list) {
8879                 if (family != NFPROTO_UNSPEC && family != table->family)
8880                         continue;
8881
8882                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
8883                         if (!nft_is_active(net, flowtable))
8884                                 goto cont;
8885                         if (idx < s_idx)
8886                                 goto cont;
8887                         if (idx > s_idx)
8888                                 memset(&cb->args[1], 0,
8889                                        sizeof(cb->args) - sizeof(cb->args[0]));
8890                         if (filter && filter->table &&
8891                             strcmp(filter->table, table->name))
8892                                 goto cont;
8893
8894                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
8895                                                           cb->nlh->nlmsg_seq,
8896                                                           NFT_MSG_NEWFLOWTABLE,
8897                                                           NLM_F_MULTI | NLM_F_APPEND,
8898                                                           table->family,
8899                                                           flowtable, NULL) < 0)
8900                                 goto done;
8901
8902                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
8903 cont:
8904                         idx++;
8905                 }
8906         }
8907 done:
8908         rcu_read_unlock();
8909
8910         cb->args[0] = idx;
8911         return skb->len;
8912 }
8913
8914 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
8915 {
8916         const struct nlattr * const *nla = cb->data;
8917         struct nft_flowtable_filter *filter = NULL;
8918
8919         if (nla[NFTA_FLOWTABLE_TABLE]) {
8920                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
8921                 if (!filter)
8922                         return -ENOMEM;
8923
8924                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
8925                                            GFP_ATOMIC);
8926                 if (!filter->table) {
8927                         kfree(filter);
8928                         return -ENOMEM;
8929                 }
8930         }
8931
8932         cb->data = filter;
8933         return 0;
8934 }
8935
8936 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
8937 {
8938         struct nft_flowtable_filter *filter = cb->data;
8939
8940         if (!filter)
8941                 return 0;
8942
8943         kfree(filter->table);
8944         kfree(filter);
8945
8946         return 0;
8947 }
8948
8949 /* called with rcu_read_lock held */
8950 static int nf_tables_getflowtable(struct sk_buff *skb,
8951                                   const struct nfnl_info *info,
8952                                   const struct nlattr * const nla[])
8953 {
8954         struct netlink_ext_ack *extack = info->extack;
8955         u8 genmask = nft_genmask_cur(info->net);
8956         u8 family = info->nfmsg->nfgen_family;
8957         struct nft_flowtable *flowtable;
8958         const struct nft_table *table;
8959         struct net *net = info->net;
8960         struct sk_buff *skb2;
8961         int err;
8962
8963         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
8964                 struct netlink_dump_control c = {
8965                         .start = nf_tables_dump_flowtable_start,
8966                         .dump = nf_tables_dump_flowtable,
8967                         .done = nf_tables_dump_flowtable_done,
8968                         .module = THIS_MODULE,
8969                         .data = (void *)nla,
8970                 };
8971
8972                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
8973         }
8974
8975         if (!nla[NFTA_FLOWTABLE_NAME])
8976                 return -EINVAL;
8977
8978         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
8979                                  genmask, 0);
8980         if (IS_ERR(table)) {
8981                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
8982                 return PTR_ERR(table);
8983         }
8984
8985         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
8986                                          genmask);
8987         if (IS_ERR(flowtable)) {
8988                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
8989                 return PTR_ERR(flowtable);
8990         }
8991
8992         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
8993         if (!skb2)
8994                 return -ENOMEM;
8995
8996         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
8997                                             info->nlh->nlmsg_seq,
8998                                             NFT_MSG_NEWFLOWTABLE, 0, family,
8999                                             flowtable, NULL);
9000         if (err < 0)
9001                 goto err_fill_flowtable_info;
9002
9003         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
9004
9005 err_fill_flowtable_info:
9006         kfree_skb(skb2);
9007         return err;
9008 }
9009
9010 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
9011                                        struct nft_flowtable *flowtable,
9012                                        struct list_head *hook_list, int event)
9013 {
9014         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
9015         struct sk_buff *skb;
9016         u16 flags = 0;
9017         int err;
9018
9019         if (!ctx->report &&
9020             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
9021                 return;
9022
9023         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
9024         if (skb == NULL)
9025                 goto err;
9026
9027         if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
9028                 flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
9029
9030         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
9031                                             ctx->seq, event, flags,
9032                                             ctx->family, flowtable, hook_list);
9033         if (err < 0) {
9034                 kfree_skb(skb);
9035                 goto err;
9036         }
9037
9038         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
9039         return;
9040 err:
9041         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
9042 }
9043
9044 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
9045 {
9046         struct nft_hook *hook, *next;
9047
9048         flowtable->data.type->free(&flowtable->data);
9049         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
9050                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
9051                                             FLOW_BLOCK_UNBIND);
9052                 list_del_rcu(&hook->list);
9053                 kfree(hook);
9054         }
9055         kfree(flowtable->name);
9056         module_put(flowtable->data.type->owner);
9057         kfree(flowtable);
9058 }
9059
9060 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
9061                                    u32 portid, u32 seq)
9062 {
9063         struct nftables_pernet *nft_net = nft_pernet(net);
9064         struct nlmsghdr *nlh;
9065         char buf[TASK_COMM_LEN];
9066         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
9067
9068         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
9069                            NFNETLINK_V0, nft_base_seq(net));
9070         if (!nlh)
9071                 goto nla_put_failure;
9072
9073         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
9074             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
9075             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
9076                 goto nla_put_failure;
9077
9078         nlmsg_end(skb, nlh);
9079         return 0;
9080
9081 nla_put_failure:
9082         nlmsg_trim(skb, nlh);
9083         return -EMSGSIZE;
9084 }
9085
9086 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
9087                                 struct nft_flowtable *flowtable)
9088 {
9089         struct nft_hook *hook;
9090
9091         list_for_each_entry(hook, &flowtable->hook_list, list) {
9092                 if (hook->ops.dev != dev)
9093                         continue;
9094
9095                 /* flow_offload_netdev_event() cleans up entries for us. */
9096                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
9097                 list_del_rcu(&hook->list);
9098                 kfree_rcu(hook, rcu);
9099                 break;
9100         }
9101 }
9102
9103 static int nf_tables_flowtable_event(struct notifier_block *this,
9104                                      unsigned long event, void *ptr)
9105 {
9106         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
9107         struct nft_flowtable *flowtable;
9108         struct nftables_pernet *nft_net;
9109         struct nft_table *table;
9110         struct net *net;
9111
9112         if (event != NETDEV_UNREGISTER)
9113                 return 0;
9114
9115         net = dev_net(dev);
9116         nft_net = nft_pernet(net);
9117         mutex_lock(&nft_net->commit_mutex);
9118         list_for_each_entry(table, &nft_net->tables, list) {
9119                 list_for_each_entry(flowtable, &table->flowtables, list) {
9120                         nft_flowtable_event(event, dev, flowtable);
9121                 }
9122         }
9123         mutex_unlock(&nft_net->commit_mutex);
9124
9125         return NOTIFY_DONE;
9126 }
9127
9128 static struct notifier_block nf_tables_flowtable_notifier = {
9129         .notifier_call  = nf_tables_flowtable_event,
9130 };
9131
9132 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
9133                                  int event)
9134 {
9135         struct nlmsghdr *nlh = nlmsg_hdr(skb);
9136         struct sk_buff *skb2;
9137         int err;
9138
9139         if (!nlmsg_report(nlh) &&
9140             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
9141                 return;
9142
9143         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
9144         if (skb2 == NULL)
9145                 goto err;
9146
9147         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
9148                                       nlh->nlmsg_seq);
9149         if (err < 0) {
9150                 kfree_skb(skb2);
9151                 goto err;
9152         }
9153
9154         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
9155                        nlmsg_report(nlh), GFP_KERNEL);
9156         return;
9157 err:
9158         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
9159                           -ENOBUFS);
9160 }
9161
9162 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
9163                             const struct nlattr * const nla[])
9164 {
9165         struct sk_buff *skb2;
9166         int err;
9167
9168         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
9169         if (skb2 == NULL)
9170                 return -ENOMEM;
9171
9172         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
9173                                       info->nlh->nlmsg_seq);
9174         if (err < 0)
9175                 goto err_fill_gen_info;
9176
9177         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
9178
9179 err_fill_gen_info:
9180         kfree_skb(skb2);
9181         return err;
9182 }
9183
9184 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
9185         [NFT_MSG_NEWTABLE] = {
9186                 .call           = nf_tables_newtable,
9187                 .type           = NFNL_CB_BATCH,
9188                 .attr_count     = NFTA_TABLE_MAX,
9189                 .policy         = nft_table_policy,
9190         },
9191         [NFT_MSG_GETTABLE] = {
9192                 .call           = nf_tables_gettable,
9193                 .type           = NFNL_CB_RCU,
9194                 .attr_count     = NFTA_TABLE_MAX,
9195                 .policy         = nft_table_policy,
9196         },
9197         [NFT_MSG_DELTABLE] = {
9198                 .call           = nf_tables_deltable,
9199                 .type           = NFNL_CB_BATCH,
9200                 .attr_count     = NFTA_TABLE_MAX,
9201                 .policy         = nft_table_policy,
9202         },
9203         [NFT_MSG_DESTROYTABLE] = {
9204                 .call           = nf_tables_deltable,
9205                 .type           = NFNL_CB_BATCH,
9206                 .attr_count     = NFTA_TABLE_MAX,
9207                 .policy         = nft_table_policy,
9208         },
9209         [NFT_MSG_NEWCHAIN] = {
9210                 .call           = nf_tables_newchain,
9211                 .type           = NFNL_CB_BATCH,
9212                 .attr_count     = NFTA_CHAIN_MAX,
9213                 .policy         = nft_chain_policy,
9214         },
9215         [NFT_MSG_GETCHAIN] = {
9216                 .call           = nf_tables_getchain,
9217                 .type           = NFNL_CB_RCU,
9218                 .attr_count     = NFTA_CHAIN_MAX,
9219                 .policy         = nft_chain_policy,
9220         },
9221         [NFT_MSG_DELCHAIN] = {
9222                 .call           = nf_tables_delchain,
9223                 .type           = NFNL_CB_BATCH,
9224                 .attr_count     = NFTA_CHAIN_MAX,
9225                 .policy         = nft_chain_policy,
9226         },
9227         [NFT_MSG_DESTROYCHAIN] = {
9228                 .call           = nf_tables_delchain,
9229                 .type           = NFNL_CB_BATCH,
9230                 .attr_count     = NFTA_CHAIN_MAX,
9231                 .policy         = nft_chain_policy,
9232         },
9233         [NFT_MSG_NEWRULE] = {
9234                 .call           = nf_tables_newrule,
9235                 .type           = NFNL_CB_BATCH,
9236                 .attr_count     = NFTA_RULE_MAX,
9237                 .policy         = nft_rule_policy,
9238         },
9239         [NFT_MSG_GETRULE] = {
9240                 .call           = nf_tables_getrule,
9241                 .type           = NFNL_CB_RCU,
9242                 .attr_count     = NFTA_RULE_MAX,
9243                 .policy         = nft_rule_policy,
9244         },
9245         [NFT_MSG_GETRULE_RESET] = {
9246                 .call           = nf_tables_getrule_reset,
9247                 .type           = NFNL_CB_RCU,
9248                 .attr_count     = NFTA_RULE_MAX,
9249                 .policy         = nft_rule_policy,
9250         },
9251         [NFT_MSG_DELRULE] = {
9252                 .call           = nf_tables_delrule,
9253                 .type           = NFNL_CB_BATCH,
9254                 .attr_count     = NFTA_RULE_MAX,
9255                 .policy         = nft_rule_policy,
9256         },
9257         [NFT_MSG_DESTROYRULE] = {
9258                 .call           = nf_tables_delrule,
9259                 .type           = NFNL_CB_BATCH,
9260                 .attr_count     = NFTA_RULE_MAX,
9261                 .policy         = nft_rule_policy,
9262         },
9263         [NFT_MSG_NEWSET] = {
9264                 .call           = nf_tables_newset,
9265                 .type           = NFNL_CB_BATCH,
9266                 .attr_count     = NFTA_SET_MAX,
9267                 .policy         = nft_set_policy,
9268         },
9269         [NFT_MSG_GETSET] = {
9270                 .call           = nf_tables_getset,
9271                 .type           = NFNL_CB_RCU,
9272                 .attr_count     = NFTA_SET_MAX,
9273                 .policy         = nft_set_policy,
9274         },
9275         [NFT_MSG_DELSET] = {
9276                 .call           = nf_tables_delset,
9277                 .type           = NFNL_CB_BATCH,
9278                 .attr_count     = NFTA_SET_MAX,
9279                 .policy         = nft_set_policy,
9280         },
9281         [NFT_MSG_DESTROYSET] = {
9282                 .call           = nf_tables_delset,
9283                 .type           = NFNL_CB_BATCH,
9284                 .attr_count     = NFTA_SET_MAX,
9285                 .policy         = nft_set_policy,
9286         },
9287         [NFT_MSG_NEWSETELEM] = {
9288                 .call           = nf_tables_newsetelem,
9289                 .type           = NFNL_CB_BATCH,
9290                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9291                 .policy         = nft_set_elem_list_policy,
9292         },
9293         [NFT_MSG_GETSETELEM] = {
9294                 .call           = nf_tables_getsetelem,
9295                 .type           = NFNL_CB_RCU,
9296                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9297                 .policy         = nft_set_elem_list_policy,
9298         },
9299         [NFT_MSG_GETSETELEM_RESET] = {
9300                 .call           = nf_tables_getsetelem_reset,
9301                 .type           = NFNL_CB_RCU,
9302                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9303                 .policy         = nft_set_elem_list_policy,
9304         },
9305         [NFT_MSG_DELSETELEM] = {
9306                 .call           = nf_tables_delsetelem,
9307                 .type           = NFNL_CB_BATCH,
9308                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9309                 .policy         = nft_set_elem_list_policy,
9310         },
9311         [NFT_MSG_DESTROYSETELEM] = {
9312                 .call           = nf_tables_delsetelem,
9313                 .type           = NFNL_CB_BATCH,
9314                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
9315                 .policy         = nft_set_elem_list_policy,
9316         },
9317         [NFT_MSG_GETGEN] = {
9318                 .call           = nf_tables_getgen,
9319                 .type           = NFNL_CB_RCU,
9320         },
9321         [NFT_MSG_NEWOBJ] = {
9322                 .call           = nf_tables_newobj,
9323                 .type           = NFNL_CB_BATCH,
9324                 .attr_count     = NFTA_OBJ_MAX,
9325                 .policy         = nft_obj_policy,
9326         },
9327         [NFT_MSG_GETOBJ] = {
9328                 .call           = nf_tables_getobj,
9329                 .type           = NFNL_CB_RCU,
9330                 .attr_count     = NFTA_OBJ_MAX,
9331                 .policy         = nft_obj_policy,
9332         },
9333         [NFT_MSG_DELOBJ] = {
9334                 .call           = nf_tables_delobj,
9335                 .type           = NFNL_CB_BATCH,
9336                 .attr_count     = NFTA_OBJ_MAX,
9337                 .policy         = nft_obj_policy,
9338         },
9339         [NFT_MSG_DESTROYOBJ] = {
9340                 .call           = nf_tables_delobj,
9341                 .type           = NFNL_CB_BATCH,
9342                 .attr_count     = NFTA_OBJ_MAX,
9343                 .policy         = nft_obj_policy,
9344         },
9345         [NFT_MSG_GETOBJ_RESET] = {
9346                 .call           = nf_tables_getobj,
9347                 .type           = NFNL_CB_RCU,
9348                 .attr_count     = NFTA_OBJ_MAX,
9349                 .policy         = nft_obj_policy,
9350         },
9351         [NFT_MSG_NEWFLOWTABLE] = {
9352                 .call           = nf_tables_newflowtable,
9353                 .type           = NFNL_CB_BATCH,
9354                 .attr_count     = NFTA_FLOWTABLE_MAX,
9355                 .policy         = nft_flowtable_policy,
9356         },
9357         [NFT_MSG_GETFLOWTABLE] = {
9358                 .call           = nf_tables_getflowtable,
9359                 .type           = NFNL_CB_RCU,
9360                 .attr_count     = NFTA_FLOWTABLE_MAX,
9361                 .policy         = nft_flowtable_policy,
9362         },
9363         [NFT_MSG_DELFLOWTABLE] = {
9364                 .call           = nf_tables_delflowtable,
9365                 .type           = NFNL_CB_BATCH,
9366                 .attr_count     = NFTA_FLOWTABLE_MAX,
9367                 .policy         = nft_flowtable_policy,
9368         },
9369         [NFT_MSG_DESTROYFLOWTABLE] = {
9370                 .call           = nf_tables_delflowtable,
9371                 .type           = NFNL_CB_BATCH,
9372                 .attr_count     = NFTA_FLOWTABLE_MAX,
9373                 .policy         = nft_flowtable_policy,
9374         },
9375 };
9376
9377 static int nf_tables_validate(struct net *net)
9378 {
9379         struct nftables_pernet *nft_net = nft_pernet(net);
9380         struct nft_table *table;
9381
9382         list_for_each_entry(table, &nft_net->tables, list) {
9383                 switch (table->validate_state) {
9384                 case NFT_VALIDATE_SKIP:
9385                         continue;
9386                 case NFT_VALIDATE_NEED:
9387                         nft_validate_state_update(table, NFT_VALIDATE_DO);
9388                         fallthrough;
9389                 case NFT_VALIDATE_DO:
9390                         if (nft_table_validate(net, table) < 0)
9391                                 return -EAGAIN;
9392
9393                         nft_validate_state_update(table, NFT_VALIDATE_SKIP);
9394                         break;
9395                 }
9396         }
9397
9398         return 0;
9399 }
9400
9401 /* a drop policy has to be deferred until all rules have been activated,
9402  * otherwise a large ruleset that contains a drop-policy base chain will
9403  * cause all packets to get dropped until the full transaction has been
9404  * processed.
9405  *
9406  * We defer the drop policy until the transaction has been finalized.
9407  */
9408 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
9409 {
9410         struct nft_base_chain *basechain;
9411
9412         if (nft_trans_chain_policy(trans) != NF_DROP)
9413                 return;
9414
9415         if (!nft_is_base_chain(trans->ctx.chain))
9416                 return;
9417
9418         basechain = nft_base_chain(trans->ctx.chain);
9419         basechain->policy = NF_DROP;
9420 }
9421
9422 static void nft_chain_commit_update(struct nft_trans *trans)
9423 {
9424         struct nft_base_chain *basechain;
9425
9426         if (nft_trans_chain_name(trans)) {
9427                 rhltable_remove(&trans->ctx.table->chains_ht,
9428                                 &trans->ctx.chain->rhlhead,
9429                                 nft_chain_ht_params);
9430                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
9431                 rhltable_insert_key(&trans->ctx.table->chains_ht,
9432                                     trans->ctx.chain->name,
9433                                     &trans->ctx.chain->rhlhead,
9434                                     nft_chain_ht_params);
9435         }
9436
9437         if (!nft_is_base_chain(trans->ctx.chain))
9438                 return;
9439
9440         nft_chain_stats_replace(trans);
9441
9442         basechain = nft_base_chain(trans->ctx.chain);
9443
9444         switch (nft_trans_chain_policy(trans)) {
9445         case NF_DROP:
9446         case NF_ACCEPT:
9447                 basechain->policy = nft_trans_chain_policy(trans);
9448                 break;
9449         }
9450 }
9451
9452 static void nft_obj_commit_update(struct nft_trans *trans)
9453 {
9454         struct nft_object *newobj;
9455         struct nft_object *obj;
9456
9457         obj = nft_trans_obj(trans);
9458         newobj = nft_trans_obj_newobj(trans);
9459
9460         if (obj->ops->update)
9461                 obj->ops->update(obj, newobj);
9462
9463         nft_obj_destroy(&trans->ctx, newobj);
9464 }
9465
9466 static void nft_commit_release(struct nft_trans *trans)
9467 {
9468         switch (trans->msg_type) {
9469         case NFT_MSG_DELTABLE:
9470         case NFT_MSG_DESTROYTABLE:
9471                 nf_tables_table_destroy(&trans->ctx);
9472                 break;
9473         case NFT_MSG_NEWCHAIN:
9474                 free_percpu(nft_trans_chain_stats(trans));
9475                 kfree(nft_trans_chain_name(trans));
9476                 break;
9477         case NFT_MSG_DELCHAIN:
9478         case NFT_MSG_DESTROYCHAIN:
9479                 if (nft_trans_chain_update(trans))
9480                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
9481                 else
9482                         nf_tables_chain_destroy(&trans->ctx);
9483                 break;
9484         case NFT_MSG_DELRULE:
9485         case NFT_MSG_DESTROYRULE:
9486                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
9487                 break;
9488         case NFT_MSG_DELSET:
9489         case NFT_MSG_DESTROYSET:
9490                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
9491                 break;
9492         case NFT_MSG_DELSETELEM:
9493         case NFT_MSG_DESTROYSETELEM:
9494                 nf_tables_set_elem_destroy(&trans->ctx,
9495                                            nft_trans_elem_set(trans),
9496                                            nft_trans_elem_priv(trans));
9497                 break;
9498         case NFT_MSG_DELOBJ:
9499         case NFT_MSG_DESTROYOBJ:
9500                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
9501                 break;
9502         case NFT_MSG_DELFLOWTABLE:
9503         case NFT_MSG_DESTROYFLOWTABLE:
9504                 if (nft_trans_flowtable_update(trans))
9505                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
9506                 else
9507                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
9508                 break;
9509         }
9510
9511         if (trans->put_net)
9512                 put_net(trans->ctx.net);
9513
9514         kfree(trans);
9515 }
9516
9517 static void nf_tables_trans_destroy_work(struct work_struct *w)
9518 {
9519         struct nft_trans *trans, *next;
9520         LIST_HEAD(head);
9521
9522         spin_lock(&nf_tables_destroy_list_lock);
9523         list_splice_init(&nf_tables_destroy_list, &head);
9524         spin_unlock(&nf_tables_destroy_list_lock);
9525
9526         if (list_empty(&head))
9527                 return;
9528
9529         synchronize_rcu();
9530
9531         list_for_each_entry_safe(trans, next, &head, list) {
9532                 nft_trans_list_del(trans);
9533                 nft_commit_release(trans);
9534         }
9535 }
9536
9537 void nf_tables_trans_destroy_flush_work(void)
9538 {
9539         flush_work(&trans_destroy_work);
9540 }
9541 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
9542
9543 static bool nft_expr_reduce(struct nft_regs_track *track,
9544                             const struct nft_expr *expr)
9545 {
9546         return false;
9547 }
9548
9549 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
9550 {
9551         const struct nft_expr *expr, *last;
9552         struct nft_regs_track track = {};
9553         unsigned int size, data_size;
9554         void *data, *data_boundary;
9555         struct nft_rule_dp *prule;
9556         struct nft_rule *rule;
9557
9558         /* already handled or inactive chain? */
9559         if (chain->blob_next || !nft_is_active_next(net, chain))
9560                 return 0;
9561
9562         data_size = 0;
9563         list_for_each_entry(rule, &chain->rules, list) {
9564                 if (nft_is_active_next(net, rule)) {
9565                         data_size += sizeof(*prule) + rule->dlen;
9566                         if (data_size > INT_MAX)
9567                                 return -ENOMEM;
9568                 }
9569         }
9570
9571         chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
9572         if (!chain->blob_next)
9573                 return -ENOMEM;
9574
9575         data = (void *)chain->blob_next->data;
9576         data_boundary = data + data_size;
9577         size = 0;
9578
9579         list_for_each_entry(rule, &chain->rules, list) {
9580                 if (!nft_is_active_next(net, rule))
9581                         continue;
9582
9583                 prule = (struct nft_rule_dp *)data;
9584                 data += offsetof(struct nft_rule_dp, data);
9585                 if (WARN_ON_ONCE(data > data_boundary))
9586                         return -ENOMEM;
9587
9588                 size = 0;
9589                 track.last = nft_expr_last(rule);
9590                 nft_rule_for_each_expr(expr, last, rule) {
9591                         track.cur = expr;
9592
9593                         if (nft_expr_reduce(&track, expr)) {
9594                                 expr = track.cur;
9595                                 continue;
9596                         }
9597
9598                         if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
9599                                 return -ENOMEM;
9600
9601                         memcpy(data + size, expr, expr->ops->size);
9602                         size += expr->ops->size;
9603                 }
9604                 if (WARN_ON_ONCE(size >= 1 << 12))
9605                         return -ENOMEM;
9606
9607                 prule->handle = rule->handle;
9608                 prule->dlen = size;
9609                 prule->is_last = 0;
9610
9611                 data += size;
9612                 size = 0;
9613                 chain->blob_next->size += (unsigned long)(data - (void *)prule);
9614         }
9615
9616         if (WARN_ON_ONCE(data > data_boundary))
9617                 return -ENOMEM;
9618
9619         prule = (struct nft_rule_dp *)data;
9620         nft_last_rule(chain, prule);
9621
9622         return 0;
9623 }
9624
9625 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
9626 {
9627         struct nftables_pernet *nft_net = nft_pernet(net);
9628         struct nft_trans *trans, *next;
9629
9630         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
9631                 struct nft_chain *chain = trans->ctx.chain;
9632
9633                 if (trans->msg_type == NFT_MSG_NEWRULE ||
9634                     trans->msg_type == NFT_MSG_DELRULE) {
9635                         kvfree(chain->blob_next);
9636                         chain->blob_next = NULL;
9637                 }
9638         }
9639 }
9640
9641 static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
9642 {
9643         struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
9644
9645         kvfree(l->blob);
9646 }
9647
9648 static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
9649 {
9650         struct nft_rule_dp_last *last;
9651
9652         /* last rule trailer is after end marker */
9653         last = (void *)blob + sizeof(*blob) + blob->size;
9654         last->blob = blob;
9655
9656         call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
9657 }
9658
9659 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
9660 {
9661         struct nft_rule_blob *g0, *g1;
9662         bool next_genbit;
9663
9664         next_genbit = nft_gencursor_next(net);
9665
9666         g0 = rcu_dereference_protected(chain->blob_gen_0,
9667                                        lockdep_commit_lock_is_held(net));
9668         g1 = rcu_dereference_protected(chain->blob_gen_1,
9669                                        lockdep_commit_lock_is_held(net));
9670
9671         /* No changes to this chain? */
9672         if (chain->blob_next == NULL) {
9673                 /* chain had no change in last or next generation */
9674                 if (g0 == g1)
9675                         return;
9676                 /*
9677                  * chain had no change in this generation; make sure next
9678                  * one uses same rules as current generation.
9679                  */
9680                 if (next_genbit) {
9681                         rcu_assign_pointer(chain->blob_gen_1, g0);
9682                         nf_tables_commit_chain_free_rules_old(g1);
9683                 } else {
9684                         rcu_assign_pointer(chain->blob_gen_0, g1);
9685                         nf_tables_commit_chain_free_rules_old(g0);
9686                 }
9687
9688                 return;
9689         }
9690
9691         if (next_genbit)
9692                 rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
9693         else
9694                 rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
9695
9696         chain->blob_next = NULL;
9697
9698         if (g0 == g1)
9699                 return;
9700
9701         if (next_genbit)
9702                 nf_tables_commit_chain_free_rules_old(g1);
9703         else
9704                 nf_tables_commit_chain_free_rules_old(g0);
9705 }
9706
9707 static void nft_obj_del(struct nft_object *obj)
9708 {
9709         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
9710         list_del_rcu(&obj->list);
9711 }
9712
9713 void nft_chain_del(struct nft_chain *chain)
9714 {
9715         struct nft_table *table = chain->table;
9716
9717         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
9718                                      nft_chain_ht_params));
9719         list_del_rcu(&chain->list);
9720 }
9721
9722 static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
9723                                         struct nft_trans_gc *trans)
9724 {
9725         struct nft_elem_priv **priv = trans->priv;
9726         unsigned int i;
9727
9728         for (i = 0; i < trans->count; i++) {
9729                 nft_setelem_data_deactivate(ctx->net, trans->set, priv[i]);
9730                 nft_setelem_remove(ctx->net, trans->set, priv[i]);
9731         }
9732 }
9733
9734 void nft_trans_gc_destroy(struct nft_trans_gc *trans)
9735 {
9736         nft_set_put(trans->set);
9737         put_net(trans->net);
9738         kfree(trans);
9739 }
9740
9741 static void nft_trans_gc_trans_free(struct rcu_head *rcu)
9742 {
9743         struct nft_elem_priv *elem_priv;
9744         struct nft_trans_gc *trans;
9745         struct nft_ctx ctx = {};
9746         unsigned int i;
9747
9748         trans = container_of(rcu, struct nft_trans_gc, rcu);
9749         ctx.net = read_pnet(&trans->set->net);
9750
9751         for (i = 0; i < trans->count; i++) {
9752                 elem_priv = trans->priv[i];
9753                 if (!nft_setelem_is_catchall(trans->set, elem_priv))
9754                         atomic_dec(&trans->set->nelems);
9755
9756                 nf_tables_set_elem_destroy(&ctx, trans->set, elem_priv);
9757         }
9758
9759         nft_trans_gc_destroy(trans);
9760 }
9761
9762 static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
9763 {
9764         struct nftables_pernet *nft_net;
9765         struct nft_ctx ctx = {};
9766
9767         nft_net = nft_pernet(trans->net);
9768
9769         mutex_lock(&nft_net->commit_mutex);
9770
9771         /* Check for race with transaction, otherwise this batch refers to
9772          * stale objects that might not be there anymore. Skip transaction if
9773          * set has been destroyed from control plane transaction in case gc
9774          * worker loses race.
9775          */
9776         if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
9777                 mutex_unlock(&nft_net->commit_mutex);
9778                 return false;
9779         }
9780
9781         ctx.net = trans->net;
9782         ctx.table = trans->set->table;
9783
9784         nft_trans_gc_setelem_remove(&ctx, trans);
9785         mutex_unlock(&nft_net->commit_mutex);
9786
9787         return true;
9788 }
9789
9790 static void nft_trans_gc_work(struct work_struct *work)
9791 {
9792         struct nft_trans_gc *trans, *next;
9793         LIST_HEAD(trans_gc_list);
9794
9795         spin_lock(&nf_tables_gc_list_lock);
9796         list_splice_init(&nf_tables_gc_list, &trans_gc_list);
9797         spin_unlock(&nf_tables_gc_list_lock);
9798
9799         list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
9800                 list_del(&trans->list);
9801                 if (!nft_trans_gc_work_done(trans)) {
9802                         nft_trans_gc_destroy(trans);
9803                         continue;
9804                 }
9805                 call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9806         }
9807 }
9808
9809 struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
9810                                         unsigned int gc_seq, gfp_t gfp)
9811 {
9812         struct net *net = read_pnet(&set->net);
9813         struct nft_trans_gc *trans;
9814
9815         trans = kzalloc(sizeof(*trans), gfp);
9816         if (!trans)
9817                 return NULL;
9818
9819         trans->net = maybe_get_net(net);
9820         if (!trans->net) {
9821                 kfree(trans);
9822                 return NULL;
9823         }
9824
9825         refcount_inc(&set->refs);
9826         trans->set = set;
9827         trans->seq = gc_seq;
9828
9829         return trans;
9830 }
9831
9832 void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
9833 {
9834         trans->priv[trans->count++] = priv;
9835 }
9836
9837 static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
9838 {
9839         spin_lock(&nf_tables_gc_list_lock);
9840         list_add_tail(&trans->list, &nf_tables_gc_list);
9841         spin_unlock(&nf_tables_gc_list_lock);
9842
9843         schedule_work(&trans_gc_work);
9844 }
9845
9846 static int nft_trans_gc_space(struct nft_trans_gc *trans)
9847 {
9848         return NFT_TRANS_GC_BATCHCOUNT - trans->count;
9849 }
9850
9851 struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
9852                                               unsigned int gc_seq, gfp_t gfp)
9853 {
9854         struct nft_set *set;
9855
9856         if (nft_trans_gc_space(gc))
9857                 return gc;
9858
9859         set = gc->set;
9860         nft_trans_gc_queue_work(gc);
9861
9862         return nft_trans_gc_alloc(set, gc_seq, gfp);
9863 }
9864
9865 void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
9866 {
9867         if (trans->count == 0) {
9868                 nft_trans_gc_destroy(trans);
9869                 return;
9870         }
9871
9872         nft_trans_gc_queue_work(trans);
9873 }
9874
9875 struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
9876 {
9877         struct nft_set *set;
9878
9879         if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
9880                 return NULL;
9881
9882         if (nft_trans_gc_space(gc))
9883                 return gc;
9884
9885         set = gc->set;
9886         call_rcu(&gc->rcu, nft_trans_gc_trans_free);
9887
9888         return nft_trans_gc_alloc(set, 0, gfp);
9889 }
9890
9891 void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
9892 {
9893         WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
9894
9895         if (trans->count == 0) {
9896                 nft_trans_gc_destroy(trans);
9897                 return;
9898         }
9899
9900         call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9901 }
9902
9903 struct nft_trans_gc *nft_trans_gc_catchall_async(struct nft_trans_gc *gc,
9904                                                  unsigned int gc_seq)
9905 {
9906         struct nft_set_elem_catchall *catchall;
9907         const struct nft_set *set = gc->set;
9908         struct nft_set_ext *ext;
9909
9910         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9911                 ext = nft_set_elem_ext(set, catchall->elem);
9912
9913                 if (!nft_set_elem_expired(ext))
9914                         continue;
9915                 if (nft_set_elem_is_dead(ext))
9916                         goto dead_elem;
9917
9918                 nft_set_elem_dead(ext);
9919 dead_elem:
9920                 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
9921                 if (!gc)
9922                         return NULL;
9923
9924                 nft_trans_gc_elem_add(gc, catchall->elem);
9925         }
9926
9927         return gc;
9928 }
9929
9930 struct nft_trans_gc *nft_trans_gc_catchall_sync(struct nft_trans_gc *gc)
9931 {
9932         struct nft_set_elem_catchall *catchall, *next;
9933         u64 tstamp = nft_net_tstamp(gc->net);
9934         const struct nft_set *set = gc->set;
9935         struct nft_elem_priv *elem_priv;
9936         struct nft_set_ext *ext;
9937
9938         WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net));
9939
9940         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
9941                 ext = nft_set_elem_ext(set, catchall->elem);
9942
9943                 if (!__nft_set_elem_expired(ext, tstamp))
9944                         continue;
9945
9946                 gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL);
9947                 if (!gc)
9948                         return NULL;
9949
9950                 elem_priv = catchall->elem;
9951                 nft_setelem_data_deactivate(gc->net, gc->set, elem_priv);
9952                 nft_setelem_catchall_destroy(catchall);
9953                 nft_trans_gc_elem_add(gc, elem_priv);
9954         }
9955
9956         return gc;
9957 }
9958
9959 static void nf_tables_module_autoload_cleanup(struct net *net)
9960 {
9961         struct nftables_pernet *nft_net = nft_pernet(net);
9962         struct nft_module_request *req, *next;
9963
9964         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
9965         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
9966                 WARN_ON_ONCE(!req->done);
9967                 list_del(&req->list);
9968                 kfree(req);
9969         }
9970 }
9971
9972 static void nf_tables_commit_release(struct net *net)
9973 {
9974         struct nftables_pernet *nft_net = nft_pernet(net);
9975         struct nft_trans *trans;
9976
9977         /* all side effects have to be made visible.
9978          * For example, if a chain named 'foo' has been deleted, a
9979          * new transaction must not find it anymore.
9980          *
9981          * Memory reclaim happens asynchronously from work queue
9982          * to prevent expensive synchronize_rcu() in commit phase.
9983          */
9984         if (list_empty(&nft_net->commit_list)) {
9985                 nf_tables_module_autoload_cleanup(net);
9986                 mutex_unlock(&nft_net->commit_mutex);
9987                 return;
9988         }
9989
9990         trans = list_last_entry(&nft_net->commit_list,
9991                                 struct nft_trans, list);
9992         get_net(trans->ctx.net);
9993         WARN_ON_ONCE(trans->put_net);
9994
9995         trans->put_net = true;
9996         spin_lock(&nf_tables_destroy_list_lock);
9997         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
9998         spin_unlock(&nf_tables_destroy_list_lock);
9999
10000         nf_tables_module_autoload_cleanup(net);
10001         schedule_work(&trans_destroy_work);
10002
10003         mutex_unlock(&nft_net->commit_mutex);
10004 }
10005
10006 static void nft_commit_notify(struct net *net, u32 portid)
10007 {
10008         struct nftables_pernet *nft_net = nft_pernet(net);
10009         struct sk_buff *batch_skb = NULL, *nskb, *skb;
10010         unsigned char *data;
10011         int len;
10012
10013         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
10014                 if (!batch_skb) {
10015 new_batch:
10016                         batch_skb = skb;
10017                         len = NLMSG_GOODSIZE - skb->len;
10018                         list_del(&skb->list);
10019                         continue;
10020                 }
10021                 len -= skb->len;
10022                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
10023                         data = skb_put(batch_skb, skb->len);
10024                         memcpy(data, skb->data, skb->len);
10025                         list_del(&skb->list);
10026                         kfree_skb(skb);
10027                         continue;
10028                 }
10029                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
10030                                NFT_CB(batch_skb).report, GFP_KERNEL);
10031                 goto new_batch;
10032         }
10033
10034         if (batch_skb) {
10035                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
10036                                NFT_CB(batch_skb).report, GFP_KERNEL);
10037         }
10038
10039         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10040 }
10041
10042 static int nf_tables_commit_audit_alloc(struct list_head *adl,
10043                                         struct nft_table *table)
10044 {
10045         struct nft_audit_data *adp;
10046
10047         list_for_each_entry(adp, adl, list) {
10048                 if (adp->table == table)
10049                         return 0;
10050         }
10051         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
10052         if (!adp)
10053                 return -ENOMEM;
10054         adp->table = table;
10055         list_add(&adp->list, adl);
10056         return 0;
10057 }
10058
10059 static void nf_tables_commit_audit_free(struct list_head *adl)
10060 {
10061         struct nft_audit_data *adp, *adn;
10062
10063         list_for_each_entry_safe(adp, adn, adl, list) {
10064                 list_del(&adp->list);
10065                 kfree(adp);
10066         }
10067 }
10068
10069 static void nf_tables_commit_audit_collect(struct list_head *adl,
10070                                            struct nft_table *table, u32 op)
10071 {
10072         struct nft_audit_data *adp;
10073
10074         list_for_each_entry(adp, adl, list) {
10075                 if (adp->table == table)
10076                         goto found;
10077         }
10078         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
10079         return;
10080 found:
10081         adp->entries++;
10082         if (!adp->op || adp->op > op)
10083                 adp->op = op;
10084 }
10085
10086 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
10087
10088 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
10089 {
10090         struct nft_audit_data *adp, *adn;
10091         char aubuf[AUNFTABLENAMELEN];
10092
10093         list_for_each_entry_safe(adp, adn, adl, list) {
10094                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
10095                          generation);
10096                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
10097                                 nft2audit_op[adp->op], GFP_KERNEL);
10098                 list_del(&adp->list);
10099                 kfree(adp);
10100         }
10101 }
10102
10103 static void nft_set_commit_update(struct list_head *set_update_list)
10104 {
10105         struct nft_set *set, *next;
10106
10107         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10108                 list_del_init(&set->pending_update);
10109
10110                 if (!set->ops->commit || set->dead)
10111                         continue;
10112
10113                 set->ops->commit(set);
10114         }
10115 }
10116
10117 static unsigned int nft_gc_seq_begin(struct nftables_pernet *nft_net)
10118 {
10119         unsigned int gc_seq;
10120
10121         /* Bump gc counter, it becomes odd, this is the busy mark. */
10122         gc_seq = READ_ONCE(nft_net->gc_seq);
10123         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10124
10125         return gc_seq;
10126 }
10127
10128 static void nft_gc_seq_end(struct nftables_pernet *nft_net, unsigned int gc_seq)
10129 {
10130         WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10131 }
10132
10133 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
10134 {
10135         struct nftables_pernet *nft_net = nft_pernet(net);
10136         struct nft_trans *trans, *next;
10137         unsigned int base_seq, gc_seq;
10138         LIST_HEAD(set_update_list);
10139         struct nft_trans_elem *te;
10140         struct nft_chain *chain;
10141         struct nft_table *table;
10142         LIST_HEAD(adl);
10143         int err;
10144
10145         if (list_empty(&nft_net->commit_list)) {
10146                 mutex_unlock(&nft_net->commit_mutex);
10147                 return 0;
10148         }
10149
10150         list_for_each_entry(trans, &nft_net->binding_list, binding_list) {
10151                 switch (trans->msg_type) {
10152                 case NFT_MSG_NEWSET:
10153                         if (!nft_trans_set_update(trans) &&
10154                             nft_set_is_anonymous(nft_trans_set(trans)) &&
10155                             !nft_trans_set_bound(trans)) {
10156                                 pr_warn_once("nftables ruleset with unbound set\n");
10157                                 return -EINVAL;
10158                         }
10159                         break;
10160                 case NFT_MSG_NEWCHAIN:
10161                         if (!nft_trans_chain_update(trans) &&
10162                             nft_chain_binding(nft_trans_chain(trans)) &&
10163                             !nft_trans_chain_bound(trans)) {
10164                                 pr_warn_once("nftables ruleset with unbound chain\n");
10165                                 return -EINVAL;
10166                         }
10167                         break;
10168                 }
10169         }
10170
10171         /* 0. Validate ruleset, otherwise roll back for error reporting. */
10172         if (nf_tables_validate(net) < 0) {
10173                 nft_net->validate_state = NFT_VALIDATE_DO;
10174                 return -EAGAIN;
10175         }
10176
10177         err = nft_flow_rule_offload_commit(net);
10178         if (err < 0)
10179                 return err;
10180
10181         /* 1.  Allocate space for next generation rules_gen_X[] */
10182         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10183                 int ret;
10184
10185                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
10186                 if (ret) {
10187                         nf_tables_commit_chain_prepare_cancel(net);
10188                         nf_tables_commit_audit_free(&adl);
10189                         return ret;
10190                 }
10191                 if (trans->msg_type == NFT_MSG_NEWRULE ||
10192                     trans->msg_type == NFT_MSG_DELRULE) {
10193                         chain = trans->ctx.chain;
10194
10195                         ret = nf_tables_commit_chain_prepare(net, chain);
10196                         if (ret < 0) {
10197                                 nf_tables_commit_chain_prepare_cancel(net);
10198                                 nf_tables_commit_audit_free(&adl);
10199                                 return ret;
10200                         }
10201                 }
10202         }
10203
10204         /* step 2.  Make rules_gen_X visible to packet path */
10205         list_for_each_entry(table, &nft_net->tables, list) {
10206                 list_for_each_entry(chain, &table->chains, list)
10207                         nf_tables_commit_chain(net, chain);
10208         }
10209
10210         /*
10211          * Bump generation counter, invalidate any dump in progress.
10212          * Cannot fail after this point.
10213          */
10214         base_seq = READ_ONCE(nft_net->base_seq);
10215         while (++base_seq == 0)
10216                 ;
10217
10218         WRITE_ONCE(nft_net->base_seq, base_seq);
10219
10220         gc_seq = nft_gc_seq_begin(nft_net);
10221
10222         /* step 3. Start new generation, rules_gen_X now in use. */
10223         net->nft.gencursor = nft_gencursor_next(net);
10224
10225         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10226                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
10227                                                trans->msg_type);
10228                 switch (trans->msg_type) {
10229                 case NFT_MSG_NEWTABLE:
10230                         if (nft_trans_table_update(trans)) {
10231                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10232                                         nft_trans_destroy(trans);
10233                                         break;
10234                                 }
10235                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
10236                                         nf_tables_table_disable(net, trans->ctx.table);
10237
10238                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10239                         } else {
10240                                 nft_clear(net, trans->ctx.table);
10241                         }
10242                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
10243                         nft_trans_destroy(trans);
10244                         break;
10245                 case NFT_MSG_DELTABLE:
10246                 case NFT_MSG_DESTROYTABLE:
10247                         list_del_rcu(&trans->ctx.table->list);
10248                         nf_tables_table_notify(&trans->ctx, trans->msg_type);
10249                         break;
10250                 case NFT_MSG_NEWCHAIN:
10251                         if (nft_trans_chain_update(trans)) {
10252                                 nft_chain_commit_update(trans);
10253                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN,
10254                                                        &nft_trans_chain_hooks(trans));
10255                                 list_splice(&nft_trans_chain_hooks(trans),
10256                                             &nft_trans_basechain(trans)->hook_list);
10257                                 /* trans destroyed after rcu grace period */
10258                         } else {
10259                                 nft_chain_commit_drop_policy(trans);
10260                                 nft_clear(net, trans->ctx.chain);
10261                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL);
10262                                 nft_trans_destroy(trans);
10263                         }
10264                         break;
10265                 case NFT_MSG_DELCHAIN:
10266                 case NFT_MSG_DESTROYCHAIN:
10267                         if (nft_trans_chain_update(trans)) {
10268                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
10269                                                        &nft_trans_chain_hooks(trans));
10270                                 if (!(trans->ctx.table->flags & NFT_TABLE_F_DORMANT)) {
10271                                         nft_netdev_unregister_hooks(net,
10272                                                                     &nft_trans_chain_hooks(trans),
10273                                                                     true);
10274                                 }
10275                         } else {
10276                                 nft_chain_del(trans->ctx.chain);
10277                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN,
10278                                                        NULL);
10279                                 nf_tables_unregister_hook(trans->ctx.net,
10280                                                           trans->ctx.table,
10281                                                           trans->ctx.chain);
10282                         }
10283                         break;
10284                 case NFT_MSG_NEWRULE:
10285                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10286                         nf_tables_rule_notify(&trans->ctx,
10287                                               nft_trans_rule(trans),
10288                                               NFT_MSG_NEWRULE);
10289                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10290                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10291
10292                         nft_trans_destroy(trans);
10293                         break;
10294                 case NFT_MSG_DELRULE:
10295                 case NFT_MSG_DESTROYRULE:
10296                         list_del_rcu(&nft_trans_rule(trans)->list);
10297                         nf_tables_rule_notify(&trans->ctx,
10298                                               nft_trans_rule(trans),
10299                                               trans->msg_type);
10300                         nft_rule_expr_deactivate(&trans->ctx,
10301                                                  nft_trans_rule(trans),
10302                                                  NFT_TRANS_COMMIT);
10303
10304                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10305                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10306                         break;
10307                 case NFT_MSG_NEWSET:
10308                         if (nft_trans_set_update(trans)) {
10309                                 struct nft_set *set = nft_trans_set(trans);
10310
10311                                 WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
10312                                 WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
10313
10314                                 if (nft_trans_set_size(trans))
10315                                         WRITE_ONCE(set->size, nft_trans_set_size(trans));
10316                         } else {
10317                                 nft_clear(net, nft_trans_set(trans));
10318                                 /* This avoids hitting -EBUSY when deleting the table
10319                                  * from the transaction.
10320                                  */
10321                                 if (nft_set_is_anonymous(nft_trans_set(trans)) &&
10322                                     !list_empty(&nft_trans_set(trans)->bindings))
10323                                         nft_use_dec(&trans->ctx.table->use);
10324                         }
10325                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10326                                              NFT_MSG_NEWSET, GFP_KERNEL);
10327                         nft_trans_destroy(trans);
10328                         break;
10329                 case NFT_MSG_DELSET:
10330                 case NFT_MSG_DESTROYSET:
10331                         nft_trans_set(trans)->dead = 1;
10332                         list_del_rcu(&nft_trans_set(trans)->list);
10333                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
10334                                              trans->msg_type, GFP_KERNEL);
10335                         break;
10336                 case NFT_MSG_NEWSETELEM:
10337                         te = (struct nft_trans_elem *)trans->data;
10338
10339                         nft_setelem_activate(net, te->set, te->elem_priv);
10340                         nf_tables_setelem_notify(&trans->ctx, te->set,
10341                                                  te->elem_priv,
10342                                                  NFT_MSG_NEWSETELEM);
10343                         if (te->set->ops->commit &&
10344                             list_empty(&te->set->pending_update)) {
10345                                 list_add_tail(&te->set->pending_update,
10346                                               &set_update_list);
10347                         }
10348                         nft_trans_destroy(trans);
10349                         break;
10350                 case NFT_MSG_DELSETELEM:
10351                 case NFT_MSG_DESTROYSETELEM:
10352                         te = (struct nft_trans_elem *)trans->data;
10353
10354                         nf_tables_setelem_notify(&trans->ctx, te->set,
10355                                                  te->elem_priv,
10356                                                  trans->msg_type);
10357                         nft_setelem_remove(net, te->set, te->elem_priv);
10358                         if (!nft_setelem_is_catchall(te->set, te->elem_priv)) {
10359                                 atomic_dec(&te->set->nelems);
10360                                 te->set->ndeact--;
10361                         }
10362                         if (te->set->ops->commit &&
10363                             list_empty(&te->set->pending_update)) {
10364                                 list_add_tail(&te->set->pending_update,
10365                                               &set_update_list);
10366                         }
10367                         break;
10368                 case NFT_MSG_NEWOBJ:
10369                         if (nft_trans_obj_update(trans)) {
10370                                 nft_obj_commit_update(trans);
10371                                 nf_tables_obj_notify(&trans->ctx,
10372                                                      nft_trans_obj(trans),
10373                                                      NFT_MSG_NEWOBJ);
10374                         } else {
10375                                 nft_clear(net, nft_trans_obj(trans));
10376                                 nf_tables_obj_notify(&trans->ctx,
10377                                                      nft_trans_obj(trans),
10378                                                      NFT_MSG_NEWOBJ);
10379                                 nft_trans_destroy(trans);
10380                         }
10381                         break;
10382                 case NFT_MSG_DELOBJ:
10383                 case NFT_MSG_DESTROYOBJ:
10384                         nft_obj_del(nft_trans_obj(trans));
10385                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
10386                                              trans->msg_type);
10387                         break;
10388                 case NFT_MSG_NEWFLOWTABLE:
10389                         if (nft_trans_flowtable_update(trans)) {
10390                                 nft_trans_flowtable(trans)->data.flags =
10391                                         nft_trans_flowtable_flags(trans);
10392                                 nf_tables_flowtable_notify(&trans->ctx,
10393                                                            nft_trans_flowtable(trans),
10394                                                            &nft_trans_flowtable_hooks(trans),
10395                                                            NFT_MSG_NEWFLOWTABLE);
10396                                 list_splice(&nft_trans_flowtable_hooks(trans),
10397                                             &nft_trans_flowtable(trans)->hook_list);
10398                         } else {
10399                                 nft_clear(net, nft_trans_flowtable(trans));
10400                                 nf_tables_flowtable_notify(&trans->ctx,
10401                                                            nft_trans_flowtable(trans),
10402                                                            NULL,
10403                                                            NFT_MSG_NEWFLOWTABLE);
10404                         }
10405                         nft_trans_destroy(trans);
10406                         break;
10407                 case NFT_MSG_DELFLOWTABLE:
10408                 case NFT_MSG_DESTROYFLOWTABLE:
10409                         if (nft_trans_flowtable_update(trans)) {
10410                                 nf_tables_flowtable_notify(&trans->ctx,
10411                                                            nft_trans_flowtable(trans),
10412                                                            &nft_trans_flowtable_hooks(trans),
10413                                                            trans->msg_type);
10414                                 nft_unregister_flowtable_net_hooks(net,
10415                                                                    &nft_trans_flowtable_hooks(trans));
10416                         } else {
10417                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10418                                 nf_tables_flowtable_notify(&trans->ctx,
10419                                                            nft_trans_flowtable(trans),
10420                                                            NULL,
10421                                                            trans->msg_type);
10422                                 nft_unregister_flowtable_net_hooks(net,
10423                                                 &nft_trans_flowtable(trans)->hook_list);
10424                         }
10425                         break;
10426                 }
10427         }
10428
10429         nft_set_commit_update(&set_update_list);
10430
10431         nft_commit_notify(net, NETLINK_CB(skb).portid);
10432         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
10433         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
10434
10435         nft_gc_seq_end(nft_net, gc_seq);
10436         nft_net->validate_state = NFT_VALIDATE_SKIP;
10437         nf_tables_commit_release(net);
10438
10439         return 0;
10440 }
10441
10442 static void nf_tables_module_autoload(struct net *net)
10443 {
10444         struct nftables_pernet *nft_net = nft_pernet(net);
10445         struct nft_module_request *req, *next;
10446         LIST_HEAD(module_list);
10447
10448         list_splice_init(&nft_net->module_list, &module_list);
10449         mutex_unlock(&nft_net->commit_mutex);
10450         list_for_each_entry_safe(req, next, &module_list, list) {
10451                 request_module("%s", req->module);
10452                 req->done = true;
10453         }
10454         mutex_lock(&nft_net->commit_mutex);
10455         list_splice(&module_list, &nft_net->module_list);
10456 }
10457
10458 static void nf_tables_abort_release(struct nft_trans *trans)
10459 {
10460         switch (trans->msg_type) {
10461         case NFT_MSG_NEWTABLE:
10462                 nf_tables_table_destroy(&trans->ctx);
10463                 break;
10464         case NFT_MSG_NEWCHAIN:
10465                 if (nft_trans_chain_update(trans))
10466                         nft_hooks_destroy(&nft_trans_chain_hooks(trans));
10467                 else
10468                         nf_tables_chain_destroy(&trans->ctx);
10469                 break;
10470         case NFT_MSG_NEWRULE:
10471                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
10472                 break;
10473         case NFT_MSG_NEWSET:
10474                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
10475                 break;
10476         case NFT_MSG_NEWSETELEM:
10477                 nft_set_elem_destroy(nft_trans_elem_set(trans),
10478                                      nft_trans_elem_priv(trans), true);
10479                 break;
10480         case NFT_MSG_NEWOBJ:
10481                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
10482                 break;
10483         case NFT_MSG_NEWFLOWTABLE:
10484                 if (nft_trans_flowtable_update(trans))
10485                         nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
10486                 else
10487                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
10488                 break;
10489         }
10490         kfree(trans);
10491 }
10492
10493 static void nft_set_abort_update(struct list_head *set_update_list)
10494 {
10495         struct nft_set *set, *next;
10496
10497         list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10498                 list_del_init(&set->pending_update);
10499
10500                 if (!set->ops->abort)
10501                         continue;
10502
10503                 set->ops->abort(set);
10504         }
10505 }
10506
10507 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
10508 {
10509         struct nftables_pernet *nft_net = nft_pernet(net);
10510         struct nft_trans *trans, *next;
10511         LIST_HEAD(set_update_list);
10512         struct nft_trans_elem *te;
10513         int err = 0;
10514
10515         if (action == NFNL_ABORT_VALIDATE &&
10516             nf_tables_validate(net) < 0)
10517                 err = -EAGAIN;
10518
10519         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
10520                                          list) {
10521                 switch (trans->msg_type) {
10522                 case NFT_MSG_NEWTABLE:
10523                         if (nft_trans_table_update(trans)) {
10524                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
10525                                         nft_trans_destroy(trans);
10526                                         break;
10527                                 }
10528                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
10529                                         nf_tables_table_disable(net, trans->ctx.table);
10530                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
10531                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
10532                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
10533                                 }
10534                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_ORPHAN) {
10535                                         trans->ctx.table->flags &= ~NFT_TABLE_F_OWNER;
10536                                         trans->ctx.table->nlpid = 0;
10537                                 }
10538                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
10539                                 nft_trans_destroy(trans);
10540                         } else {
10541                                 list_del_rcu(&trans->ctx.table->list);
10542                         }
10543                         break;
10544                 case NFT_MSG_DELTABLE:
10545                 case NFT_MSG_DESTROYTABLE:
10546                         nft_clear(trans->ctx.net, trans->ctx.table);
10547                         nft_trans_destroy(trans);
10548                         break;
10549                 case NFT_MSG_NEWCHAIN:
10550                         if (nft_trans_chain_update(trans)) {
10551                                 if (!(trans->ctx.table->flags & NFT_TABLE_F_DORMANT)) {
10552                                         nft_netdev_unregister_hooks(net,
10553                                                                     &nft_trans_chain_hooks(trans),
10554                                                                     true);
10555                                 }
10556                                 free_percpu(nft_trans_chain_stats(trans));
10557                                 kfree(nft_trans_chain_name(trans));
10558                                 nft_trans_destroy(trans);
10559                         } else {
10560                                 if (nft_trans_chain_bound(trans)) {
10561                                         nft_trans_destroy(trans);
10562                                         break;
10563                                 }
10564                                 nft_use_dec_restore(&trans->ctx.table->use);
10565                                 nft_chain_del(trans->ctx.chain);
10566                                 nf_tables_unregister_hook(trans->ctx.net,
10567                                                           trans->ctx.table,
10568                                                           trans->ctx.chain);
10569                         }
10570                         break;
10571                 case NFT_MSG_DELCHAIN:
10572                 case NFT_MSG_DESTROYCHAIN:
10573                         if (nft_trans_chain_update(trans)) {
10574                                 list_splice(&nft_trans_chain_hooks(trans),
10575                                             &nft_trans_basechain(trans)->hook_list);
10576                         } else {
10577                                 nft_use_inc_restore(&trans->ctx.table->use);
10578                                 nft_clear(trans->ctx.net, trans->ctx.chain);
10579                         }
10580                         nft_trans_destroy(trans);
10581                         break;
10582                 case NFT_MSG_NEWRULE:
10583                         if (nft_trans_rule_bound(trans)) {
10584                                 nft_trans_destroy(trans);
10585                                 break;
10586                         }
10587                         nft_use_dec_restore(&trans->ctx.chain->use);
10588                         list_del_rcu(&nft_trans_rule(trans)->list);
10589                         nft_rule_expr_deactivate(&trans->ctx,
10590                                                  nft_trans_rule(trans),
10591                                                  NFT_TRANS_ABORT);
10592                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10593                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10594                         break;
10595                 case NFT_MSG_DELRULE:
10596                 case NFT_MSG_DESTROYRULE:
10597                         nft_use_inc_restore(&trans->ctx.chain->use);
10598                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
10599                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
10600                         if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)
10601                                 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10602
10603                         nft_trans_destroy(trans);
10604                         break;
10605                 case NFT_MSG_NEWSET:
10606                         if (nft_trans_set_update(trans)) {
10607                                 nft_trans_destroy(trans);
10608                                 break;
10609                         }
10610                         nft_use_dec_restore(&trans->ctx.table->use);
10611                         if (nft_trans_set_bound(trans)) {
10612                                 nft_trans_destroy(trans);
10613                                 break;
10614                         }
10615                         nft_trans_set(trans)->dead = 1;
10616                         list_del_rcu(&nft_trans_set(trans)->list);
10617                         break;
10618                 case NFT_MSG_DELSET:
10619                 case NFT_MSG_DESTROYSET:
10620                         nft_use_inc_restore(&trans->ctx.table->use);
10621                         nft_clear(trans->ctx.net, nft_trans_set(trans));
10622                         if (nft_trans_set(trans)->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
10623                                 nft_map_activate(&trans->ctx, nft_trans_set(trans));
10624
10625                         nft_trans_destroy(trans);
10626                         break;
10627                 case NFT_MSG_NEWSETELEM:
10628                         if (nft_trans_elem_set_bound(trans)) {
10629                                 nft_trans_destroy(trans);
10630                                 break;
10631                         }
10632                         te = (struct nft_trans_elem *)trans->data;
10633                         nft_setelem_remove(net, te->set, te->elem_priv);
10634                         if (!nft_setelem_is_catchall(te->set, te->elem_priv))
10635                                 atomic_dec(&te->set->nelems);
10636
10637                         if (te->set->ops->abort &&
10638                             list_empty(&te->set->pending_update)) {
10639                                 list_add_tail(&te->set->pending_update,
10640                                               &set_update_list);
10641                         }
10642                         break;
10643                 case NFT_MSG_DELSETELEM:
10644                 case NFT_MSG_DESTROYSETELEM:
10645                         te = (struct nft_trans_elem *)trans->data;
10646
10647                         nft_setelem_data_activate(net, te->set, te->elem_priv);
10648                         nft_setelem_activate(net, te->set, te->elem_priv);
10649                         if (!nft_setelem_is_catchall(te->set, te->elem_priv))
10650                                 te->set->ndeact--;
10651
10652                         if (te->set->ops->abort &&
10653                             list_empty(&te->set->pending_update)) {
10654                                 list_add_tail(&te->set->pending_update,
10655                                               &set_update_list);
10656                         }
10657                         nft_trans_destroy(trans);
10658                         break;
10659                 case NFT_MSG_NEWOBJ:
10660                         if (nft_trans_obj_update(trans)) {
10661                                 nft_obj_destroy(&trans->ctx, nft_trans_obj_newobj(trans));
10662                                 nft_trans_destroy(trans);
10663                         } else {
10664                                 nft_use_dec_restore(&trans->ctx.table->use);
10665                                 nft_obj_del(nft_trans_obj(trans));
10666                         }
10667                         break;
10668                 case NFT_MSG_DELOBJ:
10669                 case NFT_MSG_DESTROYOBJ:
10670                         nft_use_inc_restore(&trans->ctx.table->use);
10671                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
10672                         nft_trans_destroy(trans);
10673                         break;
10674                 case NFT_MSG_NEWFLOWTABLE:
10675                         if (nft_trans_flowtable_update(trans)) {
10676                                 nft_unregister_flowtable_net_hooks(net,
10677                                                 &nft_trans_flowtable_hooks(trans));
10678                         } else {
10679                                 nft_use_dec_restore(&trans->ctx.table->use);
10680                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
10681                                 nft_unregister_flowtable_net_hooks(net,
10682                                                 &nft_trans_flowtable(trans)->hook_list);
10683                         }
10684                         break;
10685                 case NFT_MSG_DELFLOWTABLE:
10686                 case NFT_MSG_DESTROYFLOWTABLE:
10687                         if (nft_trans_flowtable_update(trans)) {
10688                                 list_splice(&nft_trans_flowtable_hooks(trans),
10689                                             &nft_trans_flowtable(trans)->hook_list);
10690                         } else {
10691                                 nft_use_inc_restore(&trans->ctx.table->use);
10692                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
10693                         }
10694                         nft_trans_destroy(trans);
10695                         break;
10696                 }
10697         }
10698
10699         nft_set_abort_update(&set_update_list);
10700
10701         synchronize_rcu();
10702
10703         list_for_each_entry_safe_reverse(trans, next,
10704                                          &nft_net->commit_list, list) {
10705                 nft_trans_list_del(trans);
10706                 nf_tables_abort_release(trans);
10707         }
10708
10709         return err;
10710 }
10711
10712 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
10713                            enum nfnl_abort_action action)
10714 {
10715         struct nftables_pernet *nft_net = nft_pernet(net);
10716         unsigned int gc_seq;
10717         int ret;
10718
10719         gc_seq = nft_gc_seq_begin(nft_net);
10720         ret = __nf_tables_abort(net, action);
10721         nft_gc_seq_end(nft_net, gc_seq);
10722
10723         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
10724
10725         /* module autoload needs to happen after GC sequence update because it
10726          * temporarily releases and grabs mutex again.
10727          */
10728         if (action == NFNL_ABORT_AUTOLOAD)
10729                 nf_tables_module_autoload(net);
10730         else
10731                 nf_tables_module_autoload_cleanup(net);
10732
10733         mutex_unlock(&nft_net->commit_mutex);
10734
10735         return ret;
10736 }
10737
10738 static bool nf_tables_valid_genid(struct net *net, u32 genid)
10739 {
10740         struct nftables_pernet *nft_net = nft_pernet(net);
10741         bool genid_ok;
10742
10743         mutex_lock(&nft_net->commit_mutex);
10744         nft_net->tstamp = get_jiffies_64();
10745
10746         genid_ok = genid == 0 || nft_net->base_seq == genid;
10747         if (!genid_ok)
10748                 mutex_unlock(&nft_net->commit_mutex);
10749
10750         /* else, commit mutex has to be released by commit or abort function */
10751         return genid_ok;
10752 }
10753
10754 static const struct nfnetlink_subsystem nf_tables_subsys = {
10755         .name           = "nf_tables",
10756         .subsys_id      = NFNL_SUBSYS_NFTABLES,
10757         .cb_count       = NFT_MSG_MAX,
10758         .cb             = nf_tables_cb,
10759         .commit         = nf_tables_commit,
10760         .abort          = nf_tables_abort,
10761         .valid_genid    = nf_tables_valid_genid,
10762         .owner          = THIS_MODULE,
10763 };
10764
10765 int nft_chain_validate_dependency(const struct nft_chain *chain,
10766                                   enum nft_chain_types type)
10767 {
10768         const struct nft_base_chain *basechain;
10769
10770         if (nft_is_base_chain(chain)) {
10771                 basechain = nft_base_chain(chain);
10772                 if (basechain->type->type != type)
10773                         return -EOPNOTSUPP;
10774         }
10775         return 0;
10776 }
10777 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
10778
10779 int nft_chain_validate_hooks(const struct nft_chain *chain,
10780                              unsigned int hook_flags)
10781 {
10782         struct nft_base_chain *basechain;
10783
10784         if (nft_is_base_chain(chain)) {
10785                 basechain = nft_base_chain(chain);
10786
10787                 if ((1 << basechain->ops.hooknum) & hook_flags)
10788                         return 0;
10789
10790                 return -EOPNOTSUPP;
10791         }
10792
10793         return 0;
10794 }
10795 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
10796
10797 /*
10798  * Loop detection - walk through the ruleset beginning at the destination chain
10799  * of a new jump until either the source chain is reached (loop) or all
10800  * reachable chains have been traversed.
10801  *
10802  * The loop check is performed whenever a new jump verdict is added to an
10803  * expression or verdict map or a verdict map is bound to a new chain.
10804  */
10805
10806 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10807                                  const struct nft_chain *chain);
10808
10809 static int nft_check_loops(const struct nft_ctx *ctx,
10810                            const struct nft_set_ext *ext)
10811 {
10812         const struct nft_data *data;
10813         int ret;
10814
10815         data = nft_set_ext_data(ext);
10816         switch (data->verdict.code) {
10817         case NFT_JUMP:
10818         case NFT_GOTO:
10819                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
10820                 break;
10821         default:
10822                 ret = 0;
10823                 break;
10824         }
10825
10826         return ret;
10827 }
10828
10829 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
10830                                         struct nft_set *set,
10831                                         const struct nft_set_iter *iter,
10832                                         struct nft_elem_priv *elem_priv)
10833 {
10834         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
10835
10836         if (!nft_set_elem_active(ext, iter->genmask))
10837                 return 0;
10838
10839         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
10840             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
10841                 return 0;
10842
10843         return nft_check_loops(ctx, ext);
10844 }
10845
10846 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
10847                                   struct nft_set *set)
10848 {
10849         u8 genmask = nft_genmask_next(ctx->net);
10850         struct nft_set_elem_catchall *catchall;
10851         struct nft_set_ext *ext;
10852         int ret = 0;
10853
10854         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10855                 ext = nft_set_elem_ext(set, catchall->elem);
10856                 if (!nft_set_elem_active(ext, genmask))
10857                         continue;
10858
10859                 ret = nft_check_loops(ctx, ext);
10860                 if (ret < 0)
10861                         return ret;
10862         }
10863
10864         return ret;
10865 }
10866
10867 static int nf_tables_check_loops(const struct nft_ctx *ctx,
10868                                  const struct nft_chain *chain)
10869 {
10870         const struct nft_rule *rule;
10871         const struct nft_expr *expr, *last;
10872         struct nft_set *set;
10873         struct nft_set_binding *binding;
10874         struct nft_set_iter iter;
10875
10876         if (ctx->chain == chain)
10877                 return -ELOOP;
10878
10879         if (fatal_signal_pending(current))
10880                 return -EINTR;
10881
10882         list_for_each_entry(rule, &chain->rules, list) {
10883                 nft_rule_for_each_expr(expr, last, rule) {
10884                         struct nft_immediate_expr *priv;
10885                         const struct nft_data *data;
10886                         int err;
10887
10888                         if (strcmp(expr->ops->type->name, "immediate"))
10889                                 continue;
10890
10891                         priv = nft_expr_priv(expr);
10892                         if (priv->dreg != NFT_REG_VERDICT)
10893                                 continue;
10894
10895                         data = &priv->data;
10896                         switch (data->verdict.code) {
10897                         case NFT_JUMP:
10898                         case NFT_GOTO:
10899                                 err = nf_tables_check_loops(ctx,
10900                                                         data->verdict.chain);
10901                                 if (err < 0)
10902                                         return err;
10903                                 break;
10904                         default:
10905                                 break;
10906                         }
10907                 }
10908         }
10909
10910         list_for_each_entry(set, &ctx->table->sets, list) {
10911                 if (!nft_is_active_next(ctx->net, set))
10912                         continue;
10913                 if (!(set->flags & NFT_SET_MAP) ||
10914                     set->dtype != NFT_DATA_VERDICT)
10915                         continue;
10916
10917                 list_for_each_entry(binding, &set->bindings, list) {
10918                         if (!(binding->flags & NFT_SET_MAP) ||
10919                             binding->chain != chain)
10920                                 continue;
10921
10922                         iter.genmask    = nft_genmask_next(ctx->net);
10923                         iter.type       = NFT_ITER_UPDATE;
10924                         iter.skip       = 0;
10925                         iter.count      = 0;
10926                         iter.err        = 0;
10927                         iter.fn         = nf_tables_loop_check_setelem;
10928
10929                         set->ops->walk(ctx, set, &iter);
10930                         if (!iter.err)
10931                                 iter.err = nft_set_catchall_loops(ctx, set);
10932
10933                         if (iter.err < 0)
10934                                 return iter.err;
10935                 }
10936         }
10937
10938         return 0;
10939 }
10940
10941 /**
10942  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
10943  *
10944  *      @attr: netlink attribute to fetch value from
10945  *      @max: maximum value to be stored in dest
10946  *      @dest: pointer to the variable
10947  *
10948  *      Parse, check and store a given u32 netlink attribute into variable.
10949  *      This function returns -ERANGE if the value goes over maximum value.
10950  *      Otherwise a 0 is returned and the attribute value is stored in the
10951  *      destination variable.
10952  */
10953 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
10954 {
10955         u32 val;
10956
10957         val = ntohl(nla_get_be32(attr));
10958         if (val > max)
10959                 return -ERANGE;
10960
10961         *dest = val;
10962         return 0;
10963 }
10964 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
10965
10966 static int nft_parse_register(const struct nlattr *attr, u32 *preg)
10967 {
10968         unsigned int reg;
10969
10970         reg = ntohl(nla_get_be32(attr));
10971         switch (reg) {
10972         case NFT_REG_VERDICT...NFT_REG_4:
10973                 *preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
10974                 break;
10975         case NFT_REG32_00...NFT_REG32_15:
10976                 *preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
10977                 break;
10978         default:
10979                 return -ERANGE;
10980         }
10981
10982         return 0;
10983 }
10984
10985 /**
10986  *      nft_dump_register - dump a register value to a netlink attribute
10987  *
10988  *      @skb: socket buffer
10989  *      @attr: attribute number
10990  *      @reg: register number
10991  *
10992  *      Construct a netlink attribute containing the register number. For
10993  *      compatibility reasons, register numbers being a multiple of 4 are
10994  *      translated to the corresponding 128 bit register numbers.
10995  */
10996 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
10997 {
10998         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
10999                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
11000         else
11001                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
11002
11003         return nla_put_be32(skb, attr, htonl(reg));
11004 }
11005 EXPORT_SYMBOL_GPL(nft_dump_register);
11006
11007 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
11008 {
11009         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
11010                 return -EINVAL;
11011         if (len == 0)
11012                 return -EINVAL;
11013         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
11014                 return -ERANGE;
11015
11016         return 0;
11017 }
11018
11019 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
11020 {
11021         u32 reg;
11022         int err;
11023
11024         err = nft_parse_register(attr, &reg);
11025         if (err < 0)
11026                 return err;
11027
11028         err = nft_validate_register_load(reg, len);
11029         if (err < 0)
11030                 return err;
11031
11032         *sreg = reg;
11033         return 0;
11034 }
11035 EXPORT_SYMBOL_GPL(nft_parse_register_load);
11036
11037 static int nft_validate_register_store(const struct nft_ctx *ctx,
11038                                        enum nft_registers reg,
11039                                        const struct nft_data *data,
11040                                        enum nft_data_types type,
11041                                        unsigned int len)
11042 {
11043         int err;
11044
11045         switch (reg) {
11046         case NFT_REG_VERDICT:
11047                 if (type != NFT_DATA_VERDICT)
11048                         return -EINVAL;
11049
11050                 if (data != NULL &&
11051                     (data->verdict.code == NFT_GOTO ||
11052                      data->verdict.code == NFT_JUMP)) {
11053                         err = nf_tables_check_loops(ctx, data->verdict.chain);
11054                         if (err < 0)
11055                                 return err;
11056                 }
11057
11058                 return 0;
11059         default:
11060                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
11061                         return -EINVAL;
11062                 if (len == 0)
11063                         return -EINVAL;
11064                 if (reg * NFT_REG32_SIZE + len >
11065                     sizeof_field(struct nft_regs, data))
11066                         return -ERANGE;
11067
11068                 if (data != NULL && type != NFT_DATA_VALUE)
11069                         return -EINVAL;
11070                 return 0;
11071         }
11072 }
11073
11074 int nft_parse_register_store(const struct nft_ctx *ctx,
11075                              const struct nlattr *attr, u8 *dreg,
11076                              const struct nft_data *data,
11077                              enum nft_data_types type, unsigned int len)
11078 {
11079         int err;
11080         u32 reg;
11081
11082         err = nft_parse_register(attr, &reg);
11083         if (err < 0)
11084                 return err;
11085
11086         err = nft_validate_register_store(ctx, reg, data, type, len);
11087         if (err < 0)
11088                 return err;
11089
11090         *dreg = reg;
11091         return 0;
11092 }
11093 EXPORT_SYMBOL_GPL(nft_parse_register_store);
11094
11095 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
11096         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
11097         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
11098                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
11099         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
11100 };
11101
11102 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
11103                             struct nft_data_desc *desc, const struct nlattr *nla)
11104 {
11105         u8 genmask = nft_genmask_next(ctx->net);
11106         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
11107         struct nft_chain *chain;
11108         int err;
11109
11110         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
11111                                           nft_verdict_policy, NULL);
11112         if (err < 0)
11113                 return err;
11114
11115         if (!tb[NFTA_VERDICT_CODE])
11116                 return -EINVAL;
11117
11118         /* zero padding hole for memcmp */
11119         memset(data, 0, sizeof(*data));
11120         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
11121
11122         switch (data->verdict.code) {
11123         case NF_ACCEPT:
11124         case NF_DROP:
11125         case NF_QUEUE:
11126                 break;
11127         case NFT_CONTINUE:
11128         case NFT_BREAK:
11129         case NFT_RETURN:
11130                 break;
11131         case NFT_JUMP:
11132         case NFT_GOTO:
11133                 if (tb[NFTA_VERDICT_CHAIN]) {
11134                         chain = nft_chain_lookup(ctx->net, ctx->table,
11135                                                  tb[NFTA_VERDICT_CHAIN],
11136                                                  genmask);
11137                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
11138                         chain = nft_chain_lookup_byid(ctx->net, ctx->table,
11139                                                       tb[NFTA_VERDICT_CHAIN_ID],
11140                                                       genmask);
11141                         if (IS_ERR(chain))
11142                                 return PTR_ERR(chain);
11143                 } else {
11144                         return -EINVAL;
11145                 }
11146
11147                 if (IS_ERR(chain))
11148                         return PTR_ERR(chain);
11149                 if (nft_is_base_chain(chain))
11150                         return -EOPNOTSUPP;
11151                 if (nft_chain_is_bound(chain))
11152                         return -EINVAL;
11153                 if (desc->flags & NFT_DATA_DESC_SETELEM &&
11154                     chain->flags & NFT_CHAIN_BINDING)
11155                         return -EINVAL;
11156                 if (!nft_use_inc(&chain->use))
11157                         return -EMFILE;
11158
11159                 data->verdict.chain = chain;
11160                 break;
11161         default:
11162                 return -EINVAL;
11163         }
11164
11165         desc->len = sizeof(data->verdict);
11166
11167         return 0;
11168 }
11169
11170 static void nft_verdict_uninit(const struct nft_data *data)
11171 {
11172         struct nft_chain *chain;
11173
11174         switch (data->verdict.code) {
11175         case NFT_JUMP:
11176         case NFT_GOTO:
11177                 chain = data->verdict.chain;
11178                 nft_use_dec(&chain->use);
11179                 break;
11180         }
11181 }
11182
11183 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
11184 {
11185         struct nlattr *nest;
11186
11187         nest = nla_nest_start_noflag(skb, type);
11188         if (!nest)
11189                 goto nla_put_failure;
11190
11191         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
11192                 goto nla_put_failure;
11193
11194         switch (v->code) {
11195         case NFT_JUMP:
11196         case NFT_GOTO:
11197                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
11198                                    v->chain->name))
11199                         goto nla_put_failure;
11200         }
11201         nla_nest_end(skb, nest);
11202         return 0;
11203
11204 nla_put_failure:
11205         return -1;
11206 }
11207
11208 static int nft_value_init(const struct nft_ctx *ctx,
11209                           struct nft_data *data, struct nft_data_desc *desc,
11210                           const struct nlattr *nla)
11211 {
11212         unsigned int len;
11213
11214         len = nla_len(nla);
11215         if (len == 0)
11216                 return -EINVAL;
11217         if (len > desc->size)
11218                 return -EOVERFLOW;
11219         if (desc->len) {
11220                 if (len != desc->len)
11221                         return -EINVAL;
11222         } else {
11223                 desc->len = len;
11224         }
11225
11226         nla_memcpy(data->data, nla, len);
11227
11228         return 0;
11229 }
11230
11231 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
11232                           unsigned int len)
11233 {
11234         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
11235 }
11236
11237 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
11238         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
11239         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
11240 };
11241
11242 /**
11243  *      nft_data_init - parse nf_tables data netlink attributes
11244  *
11245  *      @ctx: context of the expression using the data
11246  *      @data: destination struct nft_data
11247  *      @desc: data description
11248  *      @nla: netlink attribute containing data
11249  *
11250  *      Parse the netlink data attributes and initialize a struct nft_data.
11251  *      The type and length of data are returned in the data description.
11252  *
11253  *      The caller can indicate that it only wants to accept data of type
11254  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
11255  */
11256 int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
11257                   struct nft_data_desc *desc, const struct nlattr *nla)
11258 {
11259         struct nlattr *tb[NFTA_DATA_MAX + 1];
11260         int err;
11261
11262         if (WARN_ON_ONCE(!desc->size))
11263                 return -EINVAL;
11264
11265         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
11266                                           nft_data_policy, NULL);
11267         if (err < 0)
11268                 return err;
11269
11270         if (tb[NFTA_DATA_VALUE]) {
11271                 if (desc->type != NFT_DATA_VALUE)
11272                         return -EINVAL;
11273
11274                 err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
11275         } else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
11276                 if (desc->type != NFT_DATA_VERDICT)
11277                         return -EINVAL;
11278
11279                 err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
11280         } else {
11281                 err = -EINVAL;
11282         }
11283
11284         return err;
11285 }
11286 EXPORT_SYMBOL_GPL(nft_data_init);
11287
11288 /**
11289  *      nft_data_release - release a nft_data item
11290  *
11291  *      @data: struct nft_data to release
11292  *      @type: type of data
11293  *
11294  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
11295  *      all others need to be released by calling this function.
11296  */
11297 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
11298 {
11299         if (type < NFT_DATA_VERDICT)
11300                 return;
11301         switch (type) {
11302         case NFT_DATA_VERDICT:
11303                 return nft_verdict_uninit(data);
11304         default:
11305                 WARN_ON(1);
11306         }
11307 }
11308 EXPORT_SYMBOL_GPL(nft_data_release);
11309
11310 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
11311                   enum nft_data_types type, unsigned int len)
11312 {
11313         struct nlattr *nest;
11314         int err;
11315
11316         nest = nla_nest_start_noflag(skb, attr);
11317         if (nest == NULL)
11318                 return -1;
11319
11320         switch (type) {
11321         case NFT_DATA_VALUE:
11322                 err = nft_value_dump(skb, data, len);
11323                 break;
11324         case NFT_DATA_VERDICT:
11325                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
11326                 break;
11327         default:
11328                 err = -EINVAL;
11329                 WARN_ON(1);
11330         }
11331
11332         nla_nest_end(skb, nest);
11333         return err;
11334 }
11335 EXPORT_SYMBOL_GPL(nft_data_dump);
11336
11337 int __nft_release_basechain(struct nft_ctx *ctx)
11338 {
11339         struct nft_rule *rule, *nr;
11340
11341         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
11342                 return 0;
11343
11344         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
11345         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
11346                 list_del(&rule->list);
11347                 nft_use_dec(&ctx->chain->use);
11348                 nf_tables_rule_release(ctx, rule);
11349         }
11350         nft_chain_del(ctx->chain);
11351         nft_use_dec(&ctx->table->use);
11352         nf_tables_chain_destroy(ctx);
11353
11354         return 0;
11355 }
11356 EXPORT_SYMBOL_GPL(__nft_release_basechain);
11357
11358 static void __nft_release_hook(struct net *net, struct nft_table *table)
11359 {
11360         struct nft_flowtable *flowtable;
11361         struct nft_chain *chain;
11362
11363         list_for_each_entry(chain, &table->chains, list)
11364                 __nf_tables_unregister_hook(net, table, chain, true);
11365         list_for_each_entry(flowtable, &table->flowtables, list)
11366                 __nft_unregister_flowtable_net_hooks(net, &flowtable->hook_list,
11367                                                      true);
11368 }
11369
11370 static void __nft_release_hooks(struct net *net)
11371 {
11372         struct nftables_pernet *nft_net = nft_pernet(net);
11373         struct nft_table *table;
11374
11375         list_for_each_entry(table, &nft_net->tables, list) {
11376                 if (nft_table_has_owner(table))
11377                         continue;
11378
11379                 __nft_release_hook(net, table);
11380         }
11381 }
11382
11383 static void __nft_release_table(struct net *net, struct nft_table *table)
11384 {
11385         struct nft_flowtable *flowtable, *nf;
11386         struct nft_chain *chain, *nc;
11387         struct nft_object *obj, *ne;
11388         struct nft_rule *rule, *nr;
11389         struct nft_set *set, *ns;
11390         struct nft_ctx ctx = {
11391                 .net    = net,
11392                 .family = NFPROTO_NETDEV,
11393         };
11394
11395         ctx.family = table->family;
11396         ctx.table = table;
11397         list_for_each_entry(chain, &table->chains, list) {
11398                 if (nft_chain_binding(chain))
11399                         continue;
11400
11401                 ctx.chain = chain;
11402                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
11403                         list_del(&rule->list);
11404                         nft_use_dec(&chain->use);
11405                         nf_tables_rule_release(&ctx, rule);
11406                 }
11407         }
11408         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
11409                 list_del(&flowtable->list);
11410                 nft_use_dec(&table->use);
11411                 nf_tables_flowtable_destroy(flowtable);
11412         }
11413         list_for_each_entry_safe(set, ns, &table->sets, list) {
11414                 list_del(&set->list);
11415                 nft_use_dec(&table->use);
11416                 if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11417                         nft_map_deactivate(&ctx, set);
11418
11419                 nft_set_destroy(&ctx, set);
11420         }
11421         list_for_each_entry_safe(obj, ne, &table->objects, list) {
11422                 nft_obj_del(obj);
11423                 nft_use_dec(&table->use);
11424                 nft_obj_destroy(&ctx, obj);
11425         }
11426         list_for_each_entry_safe(chain, nc, &table->chains, list) {
11427                 ctx.chain = chain;
11428                 nft_chain_del(chain);
11429                 nft_use_dec(&table->use);
11430                 nf_tables_chain_destroy(&ctx);
11431         }
11432         nf_tables_table_destroy(&ctx);
11433 }
11434
11435 static void __nft_release_tables(struct net *net)
11436 {
11437         struct nftables_pernet *nft_net = nft_pernet(net);
11438         struct nft_table *table, *nt;
11439
11440         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
11441                 if (nft_table_has_owner(table))
11442                         continue;
11443
11444                 list_del(&table->list);
11445
11446                 __nft_release_table(net, table);
11447         }
11448 }
11449
11450 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
11451                             void *ptr)
11452 {
11453         struct nft_table *table, *to_delete[8];
11454         struct nftables_pernet *nft_net;
11455         struct netlink_notify *n = ptr;
11456         struct net *net = n->net;
11457         unsigned int deleted;
11458         bool restart = false;
11459         unsigned int gc_seq;
11460
11461         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
11462                 return NOTIFY_DONE;
11463
11464         nft_net = nft_pernet(net);
11465         deleted = 0;
11466         mutex_lock(&nft_net->commit_mutex);
11467
11468         gc_seq = nft_gc_seq_begin(nft_net);
11469
11470         if (!list_empty(&nf_tables_destroy_list))
11471                 nf_tables_trans_destroy_flush_work();
11472 again:
11473         list_for_each_entry(table, &nft_net->tables, list) {
11474                 if (nft_table_has_owner(table) &&
11475                     n->portid == table->nlpid) {
11476                         if (table->flags & NFT_TABLE_F_PERSIST) {
11477                                 table->flags &= ~NFT_TABLE_F_OWNER;
11478                                 continue;
11479                         }
11480                         __nft_release_hook(net, table);
11481                         list_del_rcu(&table->list);
11482                         to_delete[deleted++] = table;
11483                         if (deleted >= ARRAY_SIZE(to_delete))
11484                                 break;
11485                 }
11486         }
11487         if (deleted) {
11488                 restart = deleted >= ARRAY_SIZE(to_delete);
11489                 synchronize_rcu();
11490                 while (deleted)
11491                         __nft_release_table(net, to_delete[--deleted]);
11492
11493                 if (restart)
11494                         goto again;
11495         }
11496         nft_gc_seq_end(nft_net, gc_seq);
11497
11498         mutex_unlock(&nft_net->commit_mutex);
11499
11500         return NOTIFY_DONE;
11501 }
11502
11503 static struct notifier_block nft_nl_notifier = {
11504         .notifier_call  = nft_rcv_nl_event,
11505 };
11506
11507 static int __net_init nf_tables_init_net(struct net *net)
11508 {
11509         struct nftables_pernet *nft_net = nft_pernet(net);
11510
11511         INIT_LIST_HEAD(&nft_net->tables);
11512         INIT_LIST_HEAD(&nft_net->commit_list);
11513         INIT_LIST_HEAD(&nft_net->binding_list);
11514         INIT_LIST_HEAD(&nft_net->module_list);
11515         INIT_LIST_HEAD(&nft_net->notify_list);
11516         mutex_init(&nft_net->commit_mutex);
11517         nft_net->base_seq = 1;
11518         nft_net->gc_seq = 0;
11519         nft_net->validate_state = NFT_VALIDATE_SKIP;
11520
11521         return 0;
11522 }
11523
11524 static void __net_exit nf_tables_pre_exit_net(struct net *net)
11525 {
11526         struct nftables_pernet *nft_net = nft_pernet(net);
11527
11528         mutex_lock(&nft_net->commit_mutex);
11529         __nft_release_hooks(net);
11530         mutex_unlock(&nft_net->commit_mutex);
11531 }
11532
11533 static void __net_exit nf_tables_exit_net(struct net *net)
11534 {
11535         struct nftables_pernet *nft_net = nft_pernet(net);
11536         unsigned int gc_seq;
11537
11538         mutex_lock(&nft_net->commit_mutex);
11539
11540         gc_seq = nft_gc_seq_begin(nft_net);
11541
11542         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
11543
11544         if (!list_empty(&nft_net->module_list))
11545                 nf_tables_module_autoload_cleanup(net);
11546
11547         __nft_release_tables(net);
11548
11549         nft_gc_seq_end(nft_net, gc_seq);
11550
11551         mutex_unlock(&nft_net->commit_mutex);
11552         WARN_ON_ONCE(!list_empty(&nft_net->tables));
11553         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
11554         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
11555 }
11556
11557 static void nf_tables_exit_batch(struct list_head *net_exit_list)
11558 {
11559         flush_work(&trans_gc_work);
11560 }
11561
11562 static struct pernet_operations nf_tables_net_ops = {
11563         .init           = nf_tables_init_net,
11564         .pre_exit       = nf_tables_pre_exit_net,
11565         .exit           = nf_tables_exit_net,
11566         .exit_batch     = nf_tables_exit_batch,
11567         .id             = &nf_tables_net_id,
11568         .size           = sizeof(struct nftables_pernet),
11569 };
11570
11571 static int __init nf_tables_module_init(void)
11572 {
11573         int err;
11574
11575         err = register_pernet_subsys(&nf_tables_net_ops);
11576         if (err < 0)
11577                 return err;
11578
11579         err = nft_chain_filter_init();
11580         if (err < 0)
11581                 goto err_chain_filter;
11582
11583         err = nf_tables_core_module_init();
11584         if (err < 0)
11585                 goto err_core_module;
11586
11587         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
11588         if (err < 0)
11589                 goto err_netdev_notifier;
11590
11591         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
11592         if (err < 0)
11593                 goto err_rht_objname;
11594
11595         err = nft_offload_init();
11596         if (err < 0)
11597                 goto err_offload;
11598
11599         err = netlink_register_notifier(&nft_nl_notifier);
11600         if (err < 0)
11601                 goto err_netlink_notifier;
11602
11603         /* must be last */
11604         err = nfnetlink_subsys_register(&nf_tables_subsys);
11605         if (err < 0)
11606                 goto err_nfnl_subsys;
11607
11608         nft_chain_route_init();
11609
11610         return err;
11611
11612 err_nfnl_subsys:
11613         netlink_unregister_notifier(&nft_nl_notifier);
11614 err_netlink_notifier:
11615         nft_offload_exit();
11616 err_offload:
11617         rhltable_destroy(&nft_objname_ht);
11618 err_rht_objname:
11619         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11620 err_netdev_notifier:
11621         nf_tables_core_module_exit();
11622 err_core_module:
11623         nft_chain_filter_fini();
11624 err_chain_filter:
11625         unregister_pernet_subsys(&nf_tables_net_ops);
11626         return err;
11627 }
11628
11629 static void __exit nf_tables_module_exit(void)
11630 {
11631         nfnetlink_subsys_unregister(&nf_tables_subsys);
11632         netlink_unregister_notifier(&nft_nl_notifier);
11633         nft_offload_exit();
11634         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
11635         nft_chain_filter_fini();
11636         nft_chain_route_fini();
11637         nf_tables_trans_destroy_flush_work();
11638         unregister_pernet_subsys(&nf_tables_net_ops);
11639         cancel_work_sync(&trans_gc_work);
11640         cancel_work_sync(&trans_destroy_work);
11641         rcu_barrier();
11642         rhltable_destroy(&nft_objname_ht);
11643         nf_tables_core_module_exit();
11644 }
11645
11646 module_init(nf_tables_module_init);
11647 module_exit(nf_tables_module_exit);
11648
11649 MODULE_LICENSE("GPL");
11650 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
11651 MODULE_DESCRIPTION("Framework for packet filtering and classification");
11652 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);