net/core/devlink.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * net/core/devlink.c - Network physical/parent device Netlink interface
   4  *
   5  * Heavily inspired by net/wireless/
   6  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   7  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/module.h>
  12 #include <linux/types.h>
  13 #include <linux/slab.h>
  14 #include <linux/gfp.h>
  15 #include <linux/device.h>
  16 #include <linux/list.h>
  17 #include <linux/netdevice.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/refcount.h>
  20 #include <linux/workqueue.h>
  21 #include <linux/u64_stats_sync.h>
  22 #include <linux/timekeeping.h>
  23 #include <rdma/ib_verbs.h>
  24 #include <net/netlink.h>
  25 #include <net/genetlink.h>
  26 #include <net/rtnetlink.h>
  27 #include <net/net_namespace.h>
  28 #include <net/sock.h>
  29 #include <net/devlink.h>
  30 #define CREATE_TRACE_POINTS
  31 #include <trace/events/devlink.h>
  32
  33 static struct devlink_dpipe_field devlink_dpipe_fields_ethernet[] = {
  34         {
  35                 .name = "destination mac",
  36                 .id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC,
  37                 .bitwidth = 48,
  38         },
  39 };
  40
  41 struct devlink_dpipe_header devlink_dpipe_header_ethernet = {
  42         .name = "ethernet",
  43         .id = DEVLINK_DPIPE_HEADER_ETHERNET,
  44         .fields = devlink_dpipe_fields_ethernet,
  45         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ethernet),
  46         .global = true,
  47 };
  48 EXPORT_SYMBOL(devlink_dpipe_header_ethernet);
  49
  50 static struct devlink_dpipe_field devlink_dpipe_fields_ipv4[] = {
  51         {
  52                 .name = "destination ip",
  53                 .id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP,
  54                 .bitwidth = 32,
  55         },
  56 };
  57
  58 struct devlink_dpipe_header devlink_dpipe_header_ipv4 = {
  59         .name = "ipv4",
  60         .id = DEVLINK_DPIPE_HEADER_IPV4,
  61         .fields = devlink_dpipe_fields_ipv4,
  62         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv4),
  63         .global = true,
  64 };
  65 EXPORT_SYMBOL(devlink_dpipe_header_ipv4);
  66
  67 static struct devlink_dpipe_field devlink_dpipe_fields_ipv6[] = {
  68         {
  69                 .name = "destination ip",
  70                 .id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP,
  71                 .bitwidth = 128,
  72         },
  73 };
  74
  75 struct devlink_dpipe_header devlink_dpipe_header_ipv6 = {
  76         .name = "ipv6",
  77         .id = DEVLINK_DPIPE_HEADER_IPV6,
  78         .fields = devlink_dpipe_fields_ipv6,
  79         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv6),
  80         .global = true,
  81 };
  82 EXPORT_SYMBOL(devlink_dpipe_header_ipv6);
  83
  84 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
  85 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
  86 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
  87
  88 static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = {
  89         [DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY },
  90 };
  91
  92 static LIST_HEAD(devlink_list);
  93
  94 /* devlink_mutex
  95  *
  96  * An overall lock guarding every operation coming from userspace.
  97  * It also guards devlink devices list and it is taken when
  98  * driver registers/unregisters it.
  99  */
 100 static DEFINE_MUTEX(devlink_mutex);
 101
 102 struct net *devlink_net(const struct devlink *devlink)
 103 {
 104         return read_pnet(&devlink->_net);
 105 }
 106 EXPORT_SYMBOL_GPL(devlink_net);
 107
 108 static void __devlink_net_set(struct devlink *devlink, struct net *net)
 109 {
 110         write_pnet(&devlink->_net, net);
 111 }
 112
 113 void devlink_net_set(struct devlink *devlink, struct net *net)
 114 {
 115         if (WARN_ON(devlink->registered))
 116                 return;
 117         __devlink_net_set(devlink, net);
 118 }
 119 EXPORT_SYMBOL_GPL(devlink_net_set);
 120
 121 static struct devlink *devlink_get_from_attrs(struct net *net,
 122                                               struct nlattr **attrs)
 123 {
 124         struct devlink *devlink;
 125         char *busname;
 126         char *devname;
 127
 128         if (!attrs[DEVLINK_ATTR_BUS_NAME] || !attrs[DEVLINK_ATTR_DEV_NAME])
 129                 return ERR_PTR(-EINVAL);
 130
 131         busname = nla_data(attrs[DEVLINK_ATTR_BUS_NAME]);
 132         devname = nla_data(attrs[DEVLINK_ATTR_DEV_NAME]);
 133
 134         lockdep_assert_held(&devlink_mutex);
 135
 136         list_for_each_entry(devlink, &devlink_list, list) {
 137                 if (strcmp(devlink->dev->bus->name, busname) == 0 &&
 138                     strcmp(dev_name(devlink->dev), devname) == 0 &&
 139                     net_eq(devlink_net(devlink), net))
 140                         return devlink;
 141         }
 142
 143         return ERR_PTR(-ENODEV);
 144 }
 145
 146 static struct devlink *devlink_get_from_info(struct genl_info *info)
 147 {
 148         return devlink_get_from_attrs(genl_info_net(info), info->attrs);
 149 }
 150
 151 static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
 152                                                       unsigned int port_index)
 153 {
 154         struct devlink_port *devlink_port;
 155
 156         list_for_each_entry(devlink_port, &devlink->port_list, list) {
 157                 if (devlink_port->index == port_index)
 158                         return devlink_port;
 159         }
 160         return NULL;
 161 }
 162
 163 static bool devlink_port_index_exists(struct devlink *devlink,
 164                                       unsigned int port_index)
 165 {
 166         return devlink_port_get_by_index(devlink, port_index);
 167 }
 168
 169 static struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink,
 170                                                         struct nlattr **attrs)
 171 {
 172         if (attrs[DEVLINK_ATTR_PORT_INDEX]) {
 173                 u32 port_index = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
 174                 struct devlink_port *devlink_port;
 175
 176                 devlink_port = devlink_port_get_by_index(devlink, port_index);
 177                 if (!devlink_port)
 178                         return ERR_PTR(-ENODEV);
 179                 return devlink_port;
 180         }
 181         return ERR_PTR(-EINVAL);
 182 }
 183
 184 static struct devlink_port *devlink_port_get_from_info(struct devlink *devlink,
 185                                                        struct genl_info *info)
 186 {
 187         return devlink_port_get_from_attrs(devlink, info->attrs);
 188 }
 189
 190 struct devlink_sb {
 191         struct list_head list;
 192         unsigned int index;
 193         u32 size;
 194         u16 ingress_pools_count;
 195         u16 egress_pools_count;
 196         u16 ingress_tc_count;
 197         u16 egress_tc_count;
 198 };
 199
 200 static u16 devlink_sb_pool_count(struct devlink_sb *devlink_sb)
 201 {
 202         return devlink_sb->ingress_pools_count + devlink_sb->egress_pools_count;
 203 }
 204
 205 static struct devlink_sb *devlink_sb_get_by_index(struct devlink *devlink,
 206                                                   unsigned int sb_index)
 207 {
 208         struct devlink_sb *devlink_sb;
 209
 210         list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 211                 if (devlink_sb->index == sb_index)
 212                         return devlink_sb;
 213         }
 214         return NULL;
 215 }
 216
 217 static bool devlink_sb_index_exists(struct devlink *devlink,
 218                                     unsigned int sb_index)
 219 {
 220         return devlink_sb_get_by_index(devlink, sb_index);
 221 }
 222
 223 static struct devlink_sb *devlink_sb_get_from_attrs(struct devlink *devlink,
 224                                                     struct nlattr **attrs)
 225 {
 226         if (attrs[DEVLINK_ATTR_SB_INDEX]) {
 227                 u32 sb_index = nla_get_u32(attrs[DEVLINK_ATTR_SB_INDEX]);
 228                 struct devlink_sb *devlink_sb;
 229
 230                 devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
 231                 if (!devlink_sb)
 232                         return ERR_PTR(-ENODEV);
 233                 return devlink_sb;
 234         }
 235         return ERR_PTR(-EINVAL);
 236 }
 237
 238 static struct devlink_sb *devlink_sb_get_from_info(struct devlink *devlink,
 239                                                    struct genl_info *info)
 240 {
 241         return devlink_sb_get_from_attrs(devlink, info->attrs);
 242 }
 243
 244 static int devlink_sb_pool_index_get_from_attrs(struct devlink_sb *devlink_sb,
 245                                                 struct nlattr **attrs,
 246                                                 u16 *p_pool_index)
 247 {
 248         u16 val;
 249
 250         if (!attrs[DEVLINK_ATTR_SB_POOL_INDEX])
 251                 return -EINVAL;
 252
 253         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_POOL_INDEX]);
 254         if (val >= devlink_sb_pool_count(devlink_sb))
 255                 return -EINVAL;
 256         *p_pool_index = val;
 257         return 0;
 258 }
 259
 260 static int devlink_sb_pool_index_get_from_info(struct devlink_sb *devlink_sb,
 261                                                struct genl_info *info,
 262                                                u16 *p_pool_index)
 263 {
 264         return devlink_sb_pool_index_get_from_attrs(devlink_sb, info->attrs,
 265                                                     p_pool_index);
 266 }
 267
 268 static int
 269 devlink_sb_pool_type_get_from_attrs(struct nlattr **attrs,
 270                                     enum devlink_sb_pool_type *p_pool_type)
 271 {
 272         u8 val;
 273
 274         if (!attrs[DEVLINK_ATTR_SB_POOL_TYPE])
 275                 return -EINVAL;
 276
 277         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_TYPE]);
 278         if (val != DEVLINK_SB_POOL_TYPE_INGRESS &&
 279             val != DEVLINK_SB_POOL_TYPE_EGRESS)
 280                 return -EINVAL;
 281         *p_pool_type = val;
 282         return 0;
 283 }
 284
 285 static int
 286 devlink_sb_pool_type_get_from_info(struct genl_info *info,
 287                                    enum devlink_sb_pool_type *p_pool_type)
 288 {
 289         return devlink_sb_pool_type_get_from_attrs(info->attrs, p_pool_type);
 290 }
 291
 292 static int
 293 devlink_sb_th_type_get_from_attrs(struct nlattr **attrs,
 294                                   enum devlink_sb_threshold_type *p_th_type)
 295 {
 296         u8 val;
 297
 298         if (!attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE])
 299                 return -EINVAL;
 300
 301         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE]);
 302         if (val != DEVLINK_SB_THRESHOLD_TYPE_STATIC &&
 303             val != DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC)
 304                 return -EINVAL;
 305         *p_th_type = val;
 306         return 0;
 307 }
 308
 309 static int
 310 devlink_sb_th_type_get_from_info(struct genl_info *info,
 311                                  enum devlink_sb_threshold_type *p_th_type)
 312 {
 313         return devlink_sb_th_type_get_from_attrs(info->attrs, p_th_type);
 314 }
 315
 316 static int
 317 devlink_sb_tc_index_get_from_attrs(struct devlink_sb *devlink_sb,
 318                                    struct nlattr **attrs,
 319                                    enum devlink_sb_pool_type pool_type,
 320                                    u16 *p_tc_index)
 321 {
 322         u16 val;
 323
 324         if (!attrs[DEVLINK_ATTR_SB_TC_INDEX])
 325                 return -EINVAL;
 326
 327         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_TC_INDEX]);
 328         if (pool_type == DEVLINK_SB_POOL_TYPE_INGRESS &&
 329             val >= devlink_sb->ingress_tc_count)
 330                 return -EINVAL;
 331         if (pool_type == DEVLINK_SB_POOL_TYPE_EGRESS &&
 332             val >= devlink_sb->egress_tc_count)
 333                 return -EINVAL;
 334         *p_tc_index = val;
 335         return 0;
 336 }
 337
 338 static int
 339 devlink_sb_tc_index_get_from_info(struct devlink_sb *devlink_sb,
 340                                   struct genl_info *info,
 341                                   enum devlink_sb_pool_type pool_type,
 342                                   u16 *p_tc_index)
 343 {
 344         return devlink_sb_tc_index_get_from_attrs(devlink_sb, info->attrs,
 345                                                   pool_type, p_tc_index);
 346 }
 347
 348 struct devlink_region {
 349         struct devlink *devlink;
 350         struct devlink_port *port;
 351         struct list_head list;
 352         union {
 353                 const struct devlink_region_ops *ops;
 354                 const struct devlink_port_region_ops *port_ops;
 355         };
 356         struct list_head snapshot_list;
 357         u32 max_snapshots;
 358         u32 cur_snapshots;
 359         u64 size;
 360 };
 361
 362 struct devlink_snapshot {
 363         struct list_head list;
 364         struct devlink_region *region;
 365         u8 *data;
 366         u32 id;
 367 };
 368
 369 static struct devlink_region *
 370 devlink_region_get_by_name(struct devlink *devlink, const char *region_name)
 371 {
 372         struct devlink_region *region;
 373
 374         list_for_each_entry(region, &devlink->region_list, list)
 375                 if (!strcmp(region->ops->name, region_name))
 376                         return region;
 377
 378         return NULL;
 379 }
 380
 381 static struct devlink_region *
 382 devlink_port_region_get_by_name(struct devlink_port *port,
 383                                 const char *region_name)
 384 {
 385         struct devlink_region *region;
 386
 387         list_for_each_entry(region, &port->region_list, list)
 388                 if (!strcmp(region->ops->name, region_name))
 389                         return region;
 390
 391         return NULL;
 392 }
 393
 394 static struct devlink_snapshot *
 395 devlink_region_snapshot_get_by_id(struct devlink_region *region, u32 id)
 396 {
 397         struct devlink_snapshot *snapshot;
 398
 399         list_for_each_entry(snapshot, &region->snapshot_list, list)
 400                 if (snapshot->id == id)
 401                         return snapshot;
 402
 403         return NULL;
 404 }
 405
 406 #define DEVLINK_NL_FLAG_NEED_PORT               BIT(0)
 407 #define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT    BIT(1)
 408
 409 /* The per devlink instance lock is taken by default in the pre-doit
 410  * operation, yet several commands do not require this. The global
 411  * devlink lock is taken and protects from disruption by user-calls.
 412  */
 413 #define DEVLINK_NL_FLAG_NO_LOCK                 BIT(2)
 414
 415 static int devlink_nl_pre_doit(const struct genl_ops *ops,
 416                                struct sk_buff *skb, struct genl_info *info)
 417 {
 418         struct devlink_port *devlink_port;
 419         struct devlink *devlink;
 420         int err;
 421
 422         mutex_lock(&devlink_mutex);
 423         devlink = devlink_get_from_info(info);
 424         if (IS_ERR(devlink)) {
 425                 mutex_unlock(&devlink_mutex);
 426                 return PTR_ERR(devlink);
 427         }
 428         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 429                 mutex_lock(&devlink->lock);
 430         info->user_ptr[0] = devlink;
 431         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT) {
 432                 devlink_port = devlink_port_get_from_info(devlink, info);
 433                 if (IS_ERR(devlink_port)) {
 434                         err = PTR_ERR(devlink_port);
 435                         goto unlock;
 436                 }
 437                 info->user_ptr[1] = devlink_port;
 438         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT) {
 439                 devlink_port = devlink_port_get_from_info(devlink, info);
 440                 if (!IS_ERR(devlink_port))
 441                         info->user_ptr[1] = devlink_port;
 442         }
 443         return 0;
 444
 445 unlock:
 446         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 447                 mutex_unlock(&devlink->lock);
 448         mutex_unlock(&devlink_mutex);
 449         return err;
 450 }
 451
 452 static void devlink_nl_post_doit(const struct genl_ops *ops,
 453                                  struct sk_buff *skb, struct genl_info *info)
 454 {
 455         struct devlink *devlink;
 456
 457         devlink = info->user_ptr[0];
 458         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 459                 mutex_unlock(&devlink->lock);
 460         mutex_unlock(&devlink_mutex);
 461 }
 462
 463 static struct genl_family devlink_nl_family;
 464
 465 enum devlink_multicast_groups {
 466         DEVLINK_MCGRP_CONFIG,
 467 };
 468
 469 static const struct genl_multicast_group devlink_nl_mcgrps[] = {
 470         [DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
 471 };
 472
 473 static int devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink)
 474 {
 475         if (nla_put_string(msg, DEVLINK_ATTR_BUS_NAME, devlink->dev->bus->name))
 476                 return -EMSGSIZE;
 477         if (nla_put_string(msg, DEVLINK_ATTR_DEV_NAME, dev_name(devlink->dev)))
 478                 return -EMSGSIZE;
 479         return 0;
 480 }
 481
 482 struct devlink_reload_combination {
 483         enum devlink_reload_action action;
 484         enum devlink_reload_limit limit;
 485 };
 486
 487 static const struct devlink_reload_combination devlink_reload_invalid_combinations[] = {
 488         {
 489                 /* can't reinitialize driver with no down time */
 490                 .action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
 491                 .limit = DEVLINK_RELOAD_LIMIT_NO_RESET,
 492         },
 493 };
 494
 495 static bool
 496 devlink_reload_combination_is_invalid(enum devlink_reload_action action,
 497                                       enum devlink_reload_limit limit)
 498 {
 499         int i;
 500
 501         for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)
 502                 if (devlink_reload_invalid_combinations[i].action == action &&
 503                     devlink_reload_invalid_combinations[i].limit == limit)
 504                         return true;
 505         return false;
 506 }
 507
 508 static bool
 509 devlink_reload_action_is_supported(struct devlink *devlink, enum devlink_reload_action action)
 510 {
 511         return test_bit(action, &devlink->ops->reload_actions);
 512 }
 513
 514 static bool
 515 devlink_reload_limit_is_supported(struct devlink *devlink, enum devlink_reload_limit limit)
 516 {
 517         return test_bit(limit, &devlink->ops->reload_limits);
 518 }
 519
 520 static int devlink_reload_stat_put(struct sk_buff *msg, enum devlink_reload_action action,
 521                                    enum devlink_reload_limit limit, u32 value)
 522 {
 523         struct nlattr *reload_stats_entry;
 524
 525         reload_stats_entry = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS_ENTRY);
 526         if (!reload_stats_entry)
 527                 return -EMSGSIZE;
 528
 529         if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_ACTION, action) ||
 530             nla_put_u8(msg, DEVLINK_ATTR_RELOAD_STATS_LIMIT, limit) ||
 531             nla_put_u32(msg, DEVLINK_ATTR_RELOAD_STATS_VALUE, value))
 532                 goto nla_put_failure;
 533         nla_nest_end(msg, reload_stats_entry);
 534         return 0;
 535
 536 nla_put_failure:
 537         nla_nest_cancel(msg, reload_stats_entry);
 538         return -EMSGSIZE;
 539 }
 540
 541 static int devlink_reload_stats_put(struct sk_buff *msg, struct devlink *devlink, bool is_remote)
 542 {
 543         struct nlattr *reload_stats_attr;
 544         int i, j, stat_idx;
 545         u32 value;
 546
 547         if (!is_remote)
 548                 reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS);
 549         else
 550                 reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_REMOTE_RELOAD_STATS);
 551
 552         if (!reload_stats_attr)
 553                 return -EMSGSIZE;
 554
 555         for (j = 0; j <= DEVLINK_RELOAD_LIMIT_MAX; j++) {
 556                 /* Remote stats are shown even if not locally supported. Stats
 557                  * of actions with unspecified limit are shown though drivers
 558                  * don't need to register unspecified limit.
 559                  */
 560                 if (!is_remote && j != DEVLINK_RELOAD_LIMIT_UNSPEC &&
 561                     !devlink_reload_limit_is_supported(devlink, j))
 562                         continue;
 563                 for (i = 0; i <= DEVLINK_RELOAD_ACTION_MAX; i++) {
 564                         if ((!is_remote && !devlink_reload_action_is_supported(devlink, i)) ||
 565                             i == DEVLINK_RELOAD_ACTION_UNSPEC ||
 566                             devlink_reload_combination_is_invalid(i, j))
 567                                 continue;
 568
 569                         stat_idx = j * __DEVLINK_RELOAD_ACTION_MAX + i;
 570                         if (!is_remote)
 571                                 value = devlink->stats.reload_stats[stat_idx];
 572                         else
 573                                 value = devlink->stats.remote_reload_stats[stat_idx];
 574                         if (devlink_reload_stat_put(msg, i, j, value))
 575                                 goto nla_put_failure;
 576                 }
 577         }
 578         nla_nest_end(msg, reload_stats_attr);
 579         return 0;
 580
 581 nla_put_failure:
 582         nla_nest_cancel(msg, reload_stats_attr);
 583         return -EMSGSIZE;
 584 }
 585
 586 static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink,
 587                            enum devlink_command cmd, u32 portid,
 588                            u32 seq, int flags)
 589 {
 590         struct nlattr *dev_stats;
 591         void *hdr;
 592
 593         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 594         if (!hdr)
 595                 return -EMSGSIZE;
 596
 597         if (devlink_nl_put_handle(msg, devlink))
 598                 goto nla_put_failure;
 599         if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_FAILED, devlink->reload_failed))
 600                 goto nla_put_failure;
 601
 602         dev_stats = nla_nest_start(msg, DEVLINK_ATTR_DEV_STATS);
 603         if (!dev_stats)
 604                 goto nla_put_failure;
 605
 606         if (devlink_reload_stats_put(msg, devlink, false))
 607                 goto dev_stats_nest_cancel;
 608         if (devlink_reload_stats_put(msg, devlink, true))
 609                 goto dev_stats_nest_cancel;
 610
 611         nla_nest_end(msg, dev_stats);
 612         genlmsg_end(msg, hdr);
 613         return 0;
 614
 615 dev_stats_nest_cancel:
 616         nla_nest_cancel(msg, dev_stats);
 617 nla_put_failure:
 618         genlmsg_cancel(msg, hdr);
 619         return -EMSGSIZE;
 620 }
 621
 622 static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
 623 {
 624         struct sk_buff *msg;
 625         int err;
 626
 627         WARN_ON(cmd != DEVLINK_CMD_NEW && cmd != DEVLINK_CMD_DEL);
 628
 629         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 630         if (!msg)
 631                 return;
 632
 633         err = devlink_nl_fill(msg, devlink, cmd, 0, 0, 0);
 634         if (err) {
 635                 nlmsg_free(msg);
 636                 return;
 637         }
 638
 639         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
 640                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
 641 }
 642
 643 static int devlink_nl_port_attrs_put(struct sk_buff *msg,
 644                                      struct devlink_port *devlink_port)
 645 {
 646         struct devlink_port_attrs *attrs = &devlink_port->attrs;
 647
 648         if (!devlink_port->attrs_set)
 649                 return 0;
 650         if (attrs->lanes) {
 651                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_LANES, attrs->lanes))
 652                         return -EMSGSIZE;
 653         }
 654         if (nla_put_u8(msg, DEVLINK_ATTR_PORT_SPLITTABLE, attrs->splittable))
 655                 return -EMSGSIZE;
 656         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_FLAVOUR, attrs->flavour))
 657                 return -EMSGSIZE;
 658         switch (devlink_port->attrs.flavour) {
 659         case DEVLINK_PORT_FLAVOUR_PCI_PF:
 660                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
 661                                 attrs->pci_pf.controller) ||
 662                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_pf.pf))
 663                         return -EMSGSIZE;
 664                 if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_pf.external))
 665                         return -EMSGSIZE;
 666                 break;
 667         case DEVLINK_PORT_FLAVOUR_PCI_VF:
 668                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
 669                                 attrs->pci_vf.controller) ||
 670                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_vf.pf) ||
 671                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_VF_NUMBER, attrs->pci_vf.vf))
 672                         return -EMSGSIZE;
 673                 if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_vf.external))
 674                         return -EMSGSIZE;
 675                 break;
 676         case DEVLINK_PORT_FLAVOUR_PHYSICAL:
 677         case DEVLINK_PORT_FLAVOUR_CPU:
 678         case DEVLINK_PORT_FLAVOUR_DSA:
 679         case DEVLINK_PORT_FLAVOUR_VIRTUAL:
 680                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_NUMBER,
 681                                 attrs->phys.port_number))
 682                         return -EMSGSIZE;
 683                 if (!attrs->split)
 684                         return 0;
 685                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP,
 686                                 attrs->phys.port_number))
 687                         return -EMSGSIZE;
 688                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_SUBPORT_NUMBER,
 689                                 attrs->phys.split_subport_number))
 690                         return -EMSGSIZE;
 691                 break;
 692         default:
 693                 break;
 694         }
 695         return 0;
 696 }
 697
 698 static int
 699 devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *port,
 700                                    struct netlink_ext_ack *extack)
 701 {
 702         struct devlink *devlink = port->devlink;
 703         const struct devlink_ops *ops;
 704         struct nlattr *function_attr;
 705         bool empty_nest = true;
 706         int err = 0;
 707
 708         function_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PORT_FUNCTION);
 709         if (!function_attr)
 710                 return -EMSGSIZE;
 711
 712         ops = devlink->ops;
 713         if (ops->port_function_hw_addr_get) {
 714                 int hw_addr_len;
 715                 u8 hw_addr[MAX_ADDR_LEN];
 716
 717                 err = ops->port_function_hw_addr_get(devlink, port, hw_addr, &hw_addr_len, extack);
 718                 if (err == -EOPNOTSUPP) {
 719                         /* Port function attributes are optional for a port. If port doesn't
 720                          * support function attribute, returning -EOPNOTSUPP is not an error.
 721                          */
 722                         err = 0;
 723                         goto out;
 724                 } else if (err) {
 725                         goto out;
 726                 }
 727                 err = nla_put(msg, DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR, hw_addr_len, hw_addr);
 728                 if (err)
 729                         goto out;
 730                 empty_nest = false;
 731         }
 732
 733 out:
 734         if (err || empty_nest)
 735                 nla_nest_cancel(msg, function_attr);
 736         else
 737                 nla_nest_end(msg, function_attr);
 738         return err;
 739 }
 740
 741 static int devlink_nl_port_fill(struct sk_buff *msg, struct devlink *devlink,
 742                                 struct devlink_port *devlink_port,
 743                                 enum devlink_command cmd, u32 portid,
 744                                 u32 seq, int flags,
 745                                 struct netlink_ext_ack *extack)
 746 {
 747         void *hdr;
 748
 749         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 750         if (!hdr)
 751                 return -EMSGSIZE;
 752
 753         if (devlink_nl_put_handle(msg, devlink))
 754                 goto nla_put_failure;
 755         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
 756                 goto nla_put_failure;
 757
 758         spin_lock_bh(&devlink_port->type_lock);
 759         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
 760                 goto nla_put_failure_type_locked;
 761         if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
 762             nla_put_u16(msg, DEVLINK_ATTR_PORT_DESIRED_TYPE,
 763                         devlink_port->desired_type))
 764                 goto nla_put_failure_type_locked;
 765         if (devlink_port->type == DEVLINK_PORT_TYPE_ETH) {
 766                 struct net_device *netdev = devlink_port->type_dev;
 767
 768                 if (netdev &&
 769                     (nla_put_u32(msg, DEVLINK_ATTR_PORT_NETDEV_IFINDEX,
 770                                  netdev->ifindex) ||
 771                      nla_put_string(msg, DEVLINK_ATTR_PORT_NETDEV_NAME,
 772                                     netdev->name)))
 773                         goto nla_put_failure_type_locked;
 774         }
 775         if (devlink_port->type == DEVLINK_PORT_TYPE_IB) {
 776                 struct ib_device *ibdev = devlink_port->type_dev;
 777
 778                 if (ibdev &&
 779                     nla_put_string(msg, DEVLINK_ATTR_PORT_IBDEV_NAME,
 780                                    ibdev->name))
 781                         goto nla_put_failure_type_locked;
 782         }
 783         spin_unlock_bh(&devlink_port->type_lock);
 784         if (devlink_nl_port_attrs_put(msg, devlink_port))
 785                 goto nla_put_failure;
 786         if (devlink_nl_port_function_attrs_put(msg, devlink_port, extack))
 787                 goto nla_put_failure;
 788
 789         genlmsg_end(msg, hdr);
 790         return 0;
 791
 792 nla_put_failure_type_locked:
 793         spin_unlock_bh(&devlink_port->type_lock);
 794 nla_put_failure:
 795         genlmsg_cancel(msg, hdr);
 796         return -EMSGSIZE;
 797 }
 798
 799 static void devlink_port_notify(struct devlink_port *devlink_port,
 800                                 enum devlink_command cmd)
 801 {
 802         struct devlink *devlink = devlink_port->devlink;
 803         struct sk_buff *msg;
 804         int err;
 805
 806         if (!devlink_port->registered)
 807                 return;
 808
 809         WARN_ON(cmd != DEVLINK_CMD_PORT_NEW && cmd != DEVLINK_CMD_PORT_DEL);
 810
 811         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 812         if (!msg)
 813                 return;
 814
 815         err = devlink_nl_port_fill(msg, devlink, devlink_port, cmd, 0, 0, 0,
 816                                    NULL);
 817         if (err) {
 818                 nlmsg_free(msg);
 819                 return;
 820         }
 821
 822         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
 823                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
 824 }
 825
 826 static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info)
 827 {
 828         struct devlink *devlink = info->user_ptr[0];
 829         struct sk_buff *msg;
 830         int err;
 831
 832         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 833         if (!msg)
 834                 return -ENOMEM;
 835
 836         err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
 837                               info->snd_portid, info->snd_seq, 0);
 838         if (err) {
 839                 nlmsg_free(msg);
 840                 return err;
 841         }
 842
 843         return genlmsg_reply(msg, info);
 844 }
 845
 846 static int devlink_nl_cmd_get_dumpit(struct sk_buff *msg,
 847                                      struct netlink_callback *cb)
 848 {
 849         struct devlink *devlink;
 850         int start = cb->args[0];
 851         int idx = 0;
 852         int err;
 853
 854         mutex_lock(&devlink_mutex);
 855         list_for_each_entry(devlink, &devlink_list, list) {
 856                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
 857                         continue;
 858                 if (idx < start) {
 859                         idx++;
 860                         continue;
 861                 }
 862                 err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
 863                                       NETLINK_CB(cb->skb).portid,
 864                                       cb->nlh->nlmsg_seq, NLM_F_MULTI);
 865                 if (err)
 866                         goto out;
 867                 idx++;
 868         }
 869 out:
 870         mutex_unlock(&devlink_mutex);
 871
 872         cb->args[0] = idx;
 873         return msg->len;
 874 }
 875
 876 static int devlink_nl_cmd_port_get_doit(struct sk_buff *skb,
 877                                         struct genl_info *info)
 878 {
 879         struct devlink_port *devlink_port = info->user_ptr[1];
 880         struct devlink *devlink = devlink_port->devlink;
 881         struct sk_buff *msg;
 882         int err;
 883
 884         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 885         if (!msg)
 886                 return -ENOMEM;
 887
 888         err = devlink_nl_port_fill(msg, devlink, devlink_port,
 889                                    DEVLINK_CMD_PORT_NEW,
 890                                    info->snd_portid, info->snd_seq, 0,
 891                                    info->extack);
 892         if (err) {
 893                 nlmsg_free(msg);
 894                 return err;
 895         }
 896
 897         return genlmsg_reply(msg, info);
 898 }
 899
 900 static int devlink_nl_cmd_port_get_dumpit(struct sk_buff *msg,
 901                                           struct netlink_callback *cb)
 902 {
 903         struct devlink *devlink;
 904         struct devlink_port *devlink_port;
 905         int start = cb->args[0];
 906         int idx = 0;
 907         int err;
 908
 909         mutex_lock(&devlink_mutex);
 910         list_for_each_entry(devlink, &devlink_list, list) {
 911                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
 912                         continue;
 913                 mutex_lock(&devlink->lock);
 914                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
 915                         if (idx < start) {
 916                                 idx++;
 917                                 continue;
 918                         }
 919                         err = devlink_nl_port_fill(msg, devlink, devlink_port,
 920                                                    DEVLINK_CMD_NEW,
 921                                                    NETLINK_CB(cb->skb).portid,
 922                                                    cb->nlh->nlmsg_seq,
 923                                                    NLM_F_MULTI,
 924                                                    cb->extack);
 925                         if (err) {
 926                                 mutex_unlock(&devlink->lock);
 927                                 goto out;
 928                         }
 929                         idx++;
 930                 }
 931                 mutex_unlock(&devlink->lock);
 932         }
 933 out:
 934         mutex_unlock(&devlink_mutex);
 935
 936         cb->args[0] = idx;
 937         return msg->len;
 938 }
 939
 940 static int devlink_port_type_set(struct devlink *devlink,
 941                                  struct devlink_port *devlink_port,
 942                                  enum devlink_port_type port_type)
 943
 944 {
 945         int err;
 946
 947         if (devlink->ops->port_type_set) {
 948                 if (port_type == devlink_port->type)
 949                         return 0;
 950                 err = devlink->ops->port_type_set(devlink_port, port_type);
 951                 if (err)
 952                         return err;
 953                 devlink_port->desired_type = port_type;
 954                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
 955                 return 0;
 956         }
 957         return -EOPNOTSUPP;
 958 }
 959
 960 static int
 961 devlink_port_function_hw_addr_set(struct devlink *devlink, struct devlink_port *port,
 962                                   const struct nlattr *attr, struct netlink_ext_ack *extack)
 963 {
 964         const struct devlink_ops *ops;
 965         const u8 *hw_addr;
 966         int hw_addr_len;
 967         int err;
 968
 969         hw_addr = nla_data(attr);
 970         hw_addr_len = nla_len(attr);
 971         if (hw_addr_len > MAX_ADDR_LEN) {
 972                 NL_SET_ERR_MSG_MOD(extack, "Port function hardware address too long");
 973                 return -EINVAL;
 974         }
 975         if (port->type == DEVLINK_PORT_TYPE_ETH) {
 976                 if (hw_addr_len != ETH_ALEN) {
 977                         NL_SET_ERR_MSG_MOD(extack, "Address must be 6 bytes for Ethernet device");
 978                         return -EINVAL;
 979                 }
 980                 if (!is_unicast_ether_addr(hw_addr)) {
 981                         NL_SET_ERR_MSG_MOD(extack, "Non-unicast hardware address unsupported");
 982                         return -EINVAL;
 983                 }
 984         }
 985
 986         ops = devlink->ops;
 987         if (!ops->port_function_hw_addr_set) {
 988                 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support function attributes");
 989                 return -EOPNOTSUPP;
 990         }
 991
 992         err = ops->port_function_hw_addr_set(devlink, port, hw_addr, hw_addr_len, extack);
 993         if (err)
 994                 return err;
 995
 996         devlink_port_notify(port, DEVLINK_CMD_PORT_NEW);
 997         return 0;
 998 }
 999
1000 static int
1001 devlink_port_function_set(struct devlink *devlink, struct devlink_port *port,
1002                           const struct nlattr *attr, struct netlink_ext_ack *extack)
1003 {
1004         struct nlattr *tb[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1];
1005         int err;
1006
1007         err = nla_parse_nested(tb, DEVLINK_PORT_FUNCTION_ATTR_MAX, attr,
1008                                devlink_function_nl_policy, extack);
1009         if (err < 0) {
1010                 NL_SET_ERR_MSG_MOD(extack, "Fail to parse port function attributes");
1011                 return err;
1012         }
1013
1014         attr = tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR];
1015         if (attr)
1016                 err = devlink_port_function_hw_addr_set(devlink, port, attr, extack);
1017
1018         return err;
1019 }
1020
1021 static int devlink_nl_cmd_port_set_doit(struct sk_buff *skb,
1022                                         struct genl_info *info)
1023 {
1024         struct devlink_port *devlink_port = info->user_ptr[1];
1025         struct devlink *devlink = devlink_port->devlink;
1026         int err;
1027
1028         if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
1029                 enum devlink_port_type port_type;
1030
1031                 port_type = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_TYPE]);
1032                 err = devlink_port_type_set(devlink, devlink_port, port_type);
1033                 if (err)
1034                         return err;
1035         }
1036
1037         if (info->attrs[DEVLINK_ATTR_PORT_FUNCTION]) {
1038                 struct nlattr *attr = info->attrs[DEVLINK_ATTR_PORT_FUNCTION];
1039                 struct netlink_ext_ack *extack = info->extack;
1040
1041                 err = devlink_port_function_set(devlink, devlink_port, attr, extack);
1042                 if (err)
1043                         return err;
1044         }
1045
1046         return 0;
1047 }
1048
1049 static int devlink_port_split(struct devlink *devlink, u32 port_index,
1050                               u32 count, struct netlink_ext_ack *extack)
1051
1052 {
1053         if (devlink->ops->port_split)
1054                 return devlink->ops->port_split(devlink, port_index, count,
1055                                                 extack);
1056         return -EOPNOTSUPP;
1057 }
1058
1059 static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb,
1060                                           struct genl_info *info)
1061 {
1062         struct devlink *devlink = info->user_ptr[0];
1063         struct devlink_port *devlink_port;
1064         u32 port_index;
1065         u32 count;
1066
1067         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX] ||
1068             !info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT])
1069                 return -EINVAL;
1070
1071         devlink_port = devlink_port_get_from_info(devlink, info);
1072         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1073         count = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]);
1074
1075         if (IS_ERR(devlink_port))
1076                 return -EINVAL;
1077
1078         if (!devlink_port->attrs.splittable) {
1079                 /* Split ports cannot be split. */
1080                 if (devlink_port->attrs.split)
1081                         NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split further");
1082                 else
1083                         NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split");
1084                 return -EINVAL;
1085         }
1086
1087         if (count < 2 || !is_power_of_2(count) || count > devlink_port->attrs.lanes) {
1088                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid split count");
1089                 return -EINVAL;
1090         }
1091
1092         return devlink_port_split(devlink, port_index, count, info->extack);
1093 }
1094
1095 static int devlink_port_unsplit(struct devlink *devlink, u32 port_index,
1096                                 struct netlink_ext_ack *extack)
1097
1098 {
1099         if (devlink->ops->port_unsplit)
1100                 return devlink->ops->port_unsplit(devlink, port_index, extack);
1101         return -EOPNOTSUPP;
1102 }
1103
1104 static int devlink_nl_cmd_port_unsplit_doit(struct sk_buff *skb,
1105                                             struct genl_info *info)
1106 {
1107         struct devlink *devlink = info->user_ptr[0];
1108         u32 port_index;
1109
1110         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX])
1111                 return -EINVAL;
1112
1113         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1114         return devlink_port_unsplit(devlink, port_index, info->extack);
1115 }
1116
1117 static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink,
1118                               struct devlink_sb *devlink_sb,
1119                               enum devlink_command cmd, u32 portid,
1120                               u32 seq, int flags)
1121 {
1122         void *hdr;
1123
1124         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1125         if (!hdr)
1126                 return -EMSGSIZE;
1127
1128         if (devlink_nl_put_handle(msg, devlink))
1129                 goto nla_put_failure;
1130         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
1131                 goto nla_put_failure;
1132         if (nla_put_u32(msg, DEVLINK_ATTR_SB_SIZE, devlink_sb->size))
1133                 goto nla_put_failure;
1134         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_POOL_COUNT,
1135                         devlink_sb->ingress_pools_count))
1136                 goto nla_put_failure;
1137         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_POOL_COUNT,
1138                         devlink_sb->egress_pools_count))
1139                 goto nla_put_failure;
1140         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_TC_COUNT,
1141                         devlink_sb->ingress_tc_count))
1142                 goto nla_put_failure;
1143         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_TC_COUNT,
1144                         devlink_sb->egress_tc_count))
1145                 goto nla_put_failure;
1146
1147         genlmsg_end(msg, hdr);
1148         return 0;
1149
1150 nla_put_failure:
1151         genlmsg_cancel(msg, hdr);
1152         return -EMSGSIZE;
1153 }
1154
1155 static int devlink_nl_cmd_sb_get_doit(struct sk_buff *skb,
1156                                       struct genl_info *info)
1157 {
1158         struct devlink *devlink = info->user_ptr[0];
1159         struct devlink_sb *devlink_sb;
1160         struct sk_buff *msg;
1161         int err;
1162
1163         devlink_sb = devlink_sb_get_from_info(devlink, info);
1164         if (IS_ERR(devlink_sb))
1165                 return PTR_ERR(devlink_sb);
1166
1167         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1168         if (!msg)
1169                 return -ENOMEM;
1170
1171         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
1172                                  DEVLINK_CMD_SB_NEW,
1173                                  info->snd_portid, info->snd_seq, 0);
1174         if (err) {
1175                 nlmsg_free(msg);
1176                 return err;
1177         }
1178
1179         return genlmsg_reply(msg, info);
1180 }
1181
1182 static int devlink_nl_cmd_sb_get_dumpit(struct sk_buff *msg,
1183                                         struct netlink_callback *cb)
1184 {
1185         struct devlink *devlink;
1186         struct devlink_sb *devlink_sb;
1187         int start = cb->args[0];
1188         int idx = 0;
1189         int err;
1190
1191         mutex_lock(&devlink_mutex);
1192         list_for_each_entry(devlink, &devlink_list, list) {
1193                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
1194                         continue;
1195                 mutex_lock(&devlink->lock);
1196                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
1197                         if (idx < start) {
1198                                 idx++;
1199                                 continue;
1200                         }
1201                         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
1202                                                  DEVLINK_CMD_SB_NEW,
1203                                                  NETLINK_CB(cb->skb).portid,
1204                                                  cb->nlh->nlmsg_seq,
1205                                                  NLM_F_MULTI);
1206                         if (err) {
1207                                 mutex_unlock(&devlink->lock);
1208                                 goto out;
1209                         }
1210                         idx++;
1211                 }
1212                 mutex_unlock(&devlink->lock);
1213         }
1214 out:
1215         mutex_unlock(&devlink_mutex);
1216
1217         cb->args[0] = idx;
1218         return msg->len;
1219 }
1220
1221 static int devlink_nl_sb_pool_fill(struct sk_buff *msg, struct devlink *devlink,
1222                                    struct devlink_sb *devlink_sb,
1223                                    u16 pool_index, enum devlink_command cmd,
1224                                    u32 portid, u32 seq, int flags)
1225 {
1226         struct devlink_sb_pool_info pool_info;
1227         void *hdr;
1228         int err;
1229
1230         err = devlink->ops->sb_pool_get(devlink, devlink_sb->index,
1231                                         pool_index, &pool_info);
1232         if (err)
1233                 return err;
1234
1235         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1236         if (!hdr)
1237                 return -EMSGSIZE;
1238
1239         if (devlink_nl_put_handle(msg, devlink))
1240                 goto nla_put_failure;
1241         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
1242                 goto nla_put_failure;
1243         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
1244                 goto nla_put_failure;
1245         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_info.pool_type))
1246                 goto nla_put_failure;
1247         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_SIZE, pool_info.size))
1248                 goto nla_put_failure;
1249         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE,
1250                        pool_info.threshold_type))
1251                 goto nla_put_failure;
1252         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_CELL_SIZE,
1253                         pool_info.cell_size))
1254                 goto nla_put_failure;
1255
1256         genlmsg_end(msg, hdr);
1257         return 0;
1258
1259 nla_put_failure:
1260         genlmsg_cancel(msg, hdr);
1261         return -EMSGSIZE;
1262 }
1263
1264 static int devlink_nl_cmd_sb_pool_get_doit(struct sk_buff *skb,
1265                                            struct genl_info *info)
1266 {
1267         struct devlink *devlink = info->user_ptr[0];
1268         struct devlink_sb *devlink_sb;
1269         struct sk_buff *msg;
1270         u16 pool_index;
1271         int err;
1272
1273         devlink_sb = devlink_sb_get_from_info(devlink, info);
1274         if (IS_ERR(devlink_sb))
1275                 return PTR_ERR(devlink_sb);
1276
1277         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1278                                                   &pool_index);
1279         if (err)
1280                 return err;
1281
1282         if (!devlink->ops->sb_pool_get)
1283                 return -EOPNOTSUPP;
1284
1285         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1286         if (!msg)
1287                 return -ENOMEM;
1288
1289         err = devlink_nl_sb_pool_fill(msg, devlink, devlink_sb, pool_index,
1290                                       DEVLINK_CMD_SB_POOL_NEW,
1291                                       info->snd_portid, info->snd_seq, 0);
1292         if (err) {
1293                 nlmsg_free(msg);
1294                 return err;
1295         }
1296
1297         return genlmsg_reply(msg, info);
1298 }
1299
1300 static int __sb_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
1301                                 struct devlink *devlink,
1302                                 struct devlink_sb *devlink_sb,
1303                                 u32 portid, u32 seq)
1304 {
1305         u16 pool_count = devlink_sb_pool_count(devlink_sb);
1306         u16 pool_index;
1307         int err;
1308
1309         for (pool_index = 0; pool_index < pool_count; pool_index++) {
1310                 if (*p_idx < start) {
1311                         (*p_idx)++;
1312                         continue;
1313                 }
1314                 err = devlink_nl_sb_pool_fill(msg, devlink,
1315                                               devlink_sb,
1316                                               pool_index,
1317                                               DEVLINK_CMD_SB_POOL_NEW,
1318                                               portid, seq, NLM_F_MULTI);
1319                 if (err)
1320                         return err;
1321                 (*p_idx)++;
1322         }
1323         return 0;
1324 }
1325
1326 static int devlink_nl_cmd_sb_pool_get_dumpit(struct sk_buff *msg,
1327                                              struct netlink_callback *cb)
1328 {
1329         struct devlink *devlink;
1330         struct devlink_sb *devlink_sb;
1331         int start = cb->args[0];
1332         int idx = 0;
1333         int err = 0;
1334
1335         mutex_lock(&devlink_mutex);
1336         list_for_each_entry(devlink, &devlink_list, list) {
1337                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
1338                     !devlink->ops->sb_pool_get)
1339                         continue;
1340                 mutex_lock(&devlink->lock);
1341                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
1342                         err = __sb_pool_get_dumpit(msg, start, &idx, devlink,
1343                                                    devlink_sb,
1344                                                    NETLINK_CB(cb->skb).portid,
1345                                                    cb->nlh->nlmsg_seq);
1346                         if (err == -EOPNOTSUPP) {
1347                                 err = 0;
1348                         } else if (err) {
1349                                 mutex_unlock(&devlink->lock);
1350                                 goto out;
1351                         }
1352                 }
1353                 mutex_unlock(&devlink->lock);
1354         }
1355 out:
1356         mutex_unlock(&devlink_mutex);
1357
1358         if (err != -EMSGSIZE)
1359                 return err;
1360
1361         cb->args[0] = idx;
1362         return msg->len;
1363 }
1364
1365 static int devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
1366                                u16 pool_index, u32 size,
1367                                enum devlink_sb_threshold_type threshold_type,
1368                                struct netlink_ext_ack *extack)
1369
1370 {
1371         const struct devlink_ops *ops = devlink->ops;
1372
1373         if (ops->sb_pool_set)
1374                 return ops->sb_pool_set(devlink, sb_index, pool_index,
1375                                         size, threshold_type, extack);
1376         return -EOPNOTSUPP;
1377 }
1378
1379 static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb,
1380                                            struct genl_info *info)
1381 {
1382         struct devlink *devlink = info->user_ptr[0];
1383         enum devlink_sb_threshold_type threshold_type;
1384         struct devlink_sb *devlink_sb;
1385         u16 pool_index;
1386         u32 size;
1387         int err;
1388
1389         devlink_sb = devlink_sb_get_from_info(devlink, info);
1390         if (IS_ERR(devlink_sb))
1391                 return PTR_ERR(devlink_sb);
1392
1393         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1394                                                   &pool_index);
1395         if (err)
1396                 return err;
1397
1398         err = devlink_sb_th_type_get_from_info(info, &threshold_type);
1399         if (err)
1400                 return err;
1401
1402         if (!info->attrs[DEVLINK_ATTR_SB_POOL_SIZE])
1403                 return -EINVAL;
1404
1405         size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]);
1406         return devlink_sb_pool_set(devlink, devlink_sb->index,
1407                                    pool_index, size, threshold_type,
1408                                    info->extack);
1409 }
1410
1411 static int devlink_nl_sb_port_pool_fill(struct sk_buff *msg,
1412                                         struct devlink *devlink,
1413                                         struct devlink_port *devlink_port,
1414                                         struct devlink_sb *devlink_sb,
1415                                         u16 pool_index,
1416                                         enum devlink_command cmd,
1417                                         u32 portid, u32 seq, int flags)
1418 {
1419         const struct devlink_ops *ops = devlink->ops;
1420         u32 threshold;
1421         void *hdr;
1422         int err;
1423
1424         err = ops->sb_port_pool_get(devlink_port, devlink_sb->index,
1425                                     pool_index, &threshold);
1426         if (err)
1427                 return err;
1428
1429         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1430         if (!hdr)
1431                 return -EMSGSIZE;
1432
1433         if (devlink_nl_put_handle(msg, devlink))
1434                 goto nla_put_failure;
1435         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
1436                 goto nla_put_failure;
1437         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
1438                 goto nla_put_failure;
1439         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
1440                 goto nla_put_failure;
1441         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
1442                 goto nla_put_failure;
1443
1444         if (ops->sb_occ_port_pool_get) {
1445                 u32 cur;
1446                 u32 max;
1447
1448                 err = ops->sb_occ_port_pool_get(devlink_port, devlink_sb->index,
1449                                                 pool_index, &cur, &max);
1450                 if (err && err != -EOPNOTSUPP)
1451                         goto sb_occ_get_failure;
1452                 if (!err) {
1453                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
1454                                 goto nla_put_failure;
1455                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
1456                                 goto nla_put_failure;
1457                 }
1458         }
1459
1460         genlmsg_end(msg, hdr);
1461         return 0;
1462
1463 nla_put_failure:
1464         err = -EMSGSIZE;
1465 sb_occ_get_failure:
1466         genlmsg_cancel(msg, hdr);
1467         return err;
1468 }
1469
1470 static int devlink_nl_cmd_sb_port_pool_get_doit(struct sk_buff *skb,
1471                                                 struct genl_info *info)
1472 {
1473         struct devlink_port *devlink_port = info->user_ptr[1];
1474         struct devlink *devlink = devlink_port->devlink;
1475         struct devlink_sb *devlink_sb;
1476         struct sk_buff *msg;
1477         u16 pool_index;
1478         int err;
1479
1480         devlink_sb = devlink_sb_get_from_info(devlink, info);
1481         if (IS_ERR(devlink_sb))
1482                 return PTR_ERR(devlink_sb);
1483
1484         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1485                                                   &pool_index);
1486         if (err)
1487                 return err;
1488
1489         if (!devlink->ops->sb_port_pool_get)
1490                 return -EOPNOTSUPP;
1491
1492         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1493         if (!msg)
1494                 return -ENOMEM;
1495
1496         err = devlink_nl_sb_port_pool_fill(msg, devlink, devlink_port,
1497                                            devlink_sb, pool_index,
1498                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
1499                                            info->snd_portid, info->snd_seq, 0);
1500         if (err) {
1501                 nlmsg_free(msg);
1502                 return err;
1503         }
1504
1505         return genlmsg_reply(msg, info);
1506 }
1507
1508 static int __sb_port_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
1509                                      struct devlink *devlink,
1510                                      struct devlink_sb *devlink_sb,
1511                                      u32 portid, u32 seq)
1512 {
1513         struct devlink_port *devlink_port;
1514         u16 pool_count = devlink_sb_pool_count(devlink_sb);
1515         u16 pool_index;
1516         int err;
1517
1518         list_for_each_entry(devlink_port, &devlink->port_list, list) {
1519                 for (pool_index = 0; pool_index < pool_count; pool_index++) {
1520                         if (*p_idx < start) {
1521                                 (*p_idx)++;
1522                                 continue;
1523                         }
1524                         err = devlink_nl_sb_port_pool_fill(msg, devlink,
1525                                                            devlink_port,
1526                                                            devlink_sb,
1527                                                            pool_index,
1528                                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
1529                                                            portid, seq,
1530                                                            NLM_F_MULTI);
1531                         if (err)
1532                                 return err;
1533                         (*p_idx)++;
1534                 }
1535         }
1536         return 0;
1537 }
1538
1539 static int devlink_nl_cmd_sb_port_pool_get_dumpit(struct sk_buff *msg,
1540                                                   struct netlink_callback *cb)
1541 {
1542         struct devlink *devlink;
1543         struct devlink_sb *devlink_sb;
1544         int start = cb->args[0];
1545         int idx = 0;
1546         int err = 0;
1547
1548         mutex_lock(&devlink_mutex);
1549         list_for_each_entry(devlink, &devlink_list, list) {
1550                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
1551                     !devlink->ops->sb_port_pool_get)
1552                         continue;
1553                 mutex_lock(&devlink->lock);
1554                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
1555                         err = __sb_port_pool_get_dumpit(msg, start, &idx,
1556                                                         devlink, devlink_sb,
1557                                                         NETLINK_CB(cb->skb).portid,
1558                                                         cb->nlh->nlmsg_seq);
1559                         if (err == -EOPNOTSUPP) {
1560                                 err = 0;
1561                         } else if (err) {
1562                                 mutex_unlock(&devlink->lock);
1563                                 goto out;
1564                         }
1565                 }
1566                 mutex_unlock(&devlink->lock);
1567         }
1568 out:
1569         mutex_unlock(&devlink_mutex);
1570
1571         if (err != -EMSGSIZE)
1572                 return err;
1573
1574         cb->args[0] = idx;
1575         return msg->len;
1576 }
1577
1578 static int devlink_sb_port_pool_set(struct devlink_port *devlink_port,
1579                                     unsigned int sb_index, u16 pool_index,
1580                                     u32 threshold,
1581                                     struct netlink_ext_ack *extack)
1582
1583 {
1584         const struct devlink_ops *ops = devlink_port->devlink->ops;
1585
1586         if (ops->sb_port_pool_set)
1587                 return ops->sb_port_pool_set(devlink_port, sb_index,
1588                                              pool_index, threshold, extack);
1589         return -EOPNOTSUPP;
1590 }
1591
1592 static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb,
1593                                                 struct genl_info *info)
1594 {
1595         struct devlink_port *devlink_port = info->user_ptr[1];
1596         struct devlink *devlink = info->user_ptr[0];
1597         struct devlink_sb *devlink_sb;
1598         u16 pool_index;
1599         u32 threshold;
1600         int err;
1601
1602         devlink_sb = devlink_sb_get_from_info(devlink, info);
1603         if (IS_ERR(devlink_sb))
1604                 return PTR_ERR(devlink_sb);
1605
1606         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1607                                                   &pool_index);
1608         if (err)
1609                 return err;
1610
1611         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
1612                 return -EINVAL;
1613
1614         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
1615         return devlink_sb_port_pool_set(devlink_port, devlink_sb->index,
1616                                         pool_index, threshold, info->extack);
1617 }
1618
1619 static int
1620 devlink_nl_sb_tc_pool_bind_fill(struct sk_buff *msg, struct devlink *devlink,
1621                                 struct devlink_port *devlink_port,
1622                                 struct devlink_sb *devlink_sb, u16 tc_index,
1623                                 enum devlink_sb_pool_type pool_type,
1624                                 enum devlink_command cmd,
1625                                 u32 portid, u32 seq, int flags)
1626 {
1627         const struct devlink_ops *ops = devlink->ops;
1628         u16 pool_index;
1629         u32 threshold;
1630         void *hdr;
1631         int err;
1632
1633         err = ops->sb_tc_pool_bind_get(devlink_port, devlink_sb->index,
1634                                        tc_index, pool_type,
1635                                        &pool_index, &threshold);
1636         if (err)
1637                 return err;
1638
1639         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1640         if (!hdr)
1641                 return -EMSGSIZE;
1642
1643         if (devlink_nl_put_handle(msg, devlink))
1644                 goto nla_put_failure;
1645         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
1646                 goto nla_put_failure;
1647         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
1648                 goto nla_put_failure;
1649         if (nla_put_u16(msg, DEVLINK_ATTR_SB_TC_INDEX, tc_index))
1650                 goto nla_put_failure;
1651         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_type))
1652                 goto nla_put_failure;
1653         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
1654                 goto nla_put_failure;
1655         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
1656                 goto nla_put_failure;
1657
1658         if (ops->sb_occ_tc_port_bind_get) {
1659                 u32 cur;
1660                 u32 max;
1661
1662                 err = ops->sb_occ_tc_port_bind_get(devlink_port,
1663                                                    devlink_sb->index,
1664                                                    tc_index, pool_type,
1665                                                    &cur, &max);
1666                 if (err && err != -EOPNOTSUPP)
1667                         return err;
1668                 if (!err) {
1669                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
1670                                 goto nla_put_failure;
1671                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
1672                                 goto nla_put_failure;
1673                 }
1674         }
1675
1676         genlmsg_end(msg, hdr);
1677         return 0;
1678
1679 nla_put_failure:
1680         genlmsg_cancel(msg, hdr);
1681         return -EMSGSIZE;
1682 }
1683
1684 static int devlink_nl_cmd_sb_tc_pool_bind_get_doit(struct sk_buff *skb,
1685                                                    struct genl_info *info)
1686 {
1687         struct devlink_port *devlink_port = info->user_ptr[1];
1688         struct devlink *devlink = devlink_port->devlink;
1689         struct devlink_sb *devlink_sb;
1690         struct sk_buff *msg;
1691         enum devlink_sb_pool_type pool_type;
1692         u16 tc_index;
1693         int err;
1694
1695         devlink_sb = devlink_sb_get_from_info(devlink, info);
1696         if (IS_ERR(devlink_sb))
1697                 return PTR_ERR(devlink_sb);
1698
1699         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
1700         if (err)
1701                 return err;
1702
1703         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
1704                                                 pool_type, &tc_index);
1705         if (err)
1706                 return err;
1707
1708         if (!devlink->ops->sb_tc_pool_bind_get)
1709                 return -EOPNOTSUPP;
1710
1711         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1712         if (!msg)
1713                 return -ENOMEM;
1714
1715         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink, devlink_port,
1716                                               devlink_sb, tc_index, pool_type,
1717                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
1718                                               info->snd_portid,
1719                                               info->snd_seq, 0);
1720         if (err) {
1721                 nlmsg_free(msg);
1722                 return err;
1723         }
1724
1725         return genlmsg_reply(msg, info);
1726 }
1727
1728 static int __sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
1729                                         int start, int *p_idx,
1730                                         struct devlink *devlink,
1731                                         struct devlink_sb *devlink_sb,
1732                                         u32 portid, u32 seq)
1733 {
1734         struct devlink_port *devlink_port;
1735         u16 tc_index;
1736         int err;
1737
1738         list_for_each_entry(devlink_port, &devlink->port_list, list) {
1739                 for (tc_index = 0;
1740                      tc_index < devlink_sb->ingress_tc_count; tc_index++) {
1741                         if (*p_idx < start) {
1742                                 (*p_idx)++;
1743                                 continue;
1744                         }
1745                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
1746                                                               devlink_port,
1747                                                               devlink_sb,
1748                                                               tc_index,
1749                                                               DEVLINK_SB_POOL_TYPE_INGRESS,
1750                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
1751                                                               portid, seq,
1752                                                               NLM_F_MULTI);
1753                         if (err)
1754                                 return err;
1755                         (*p_idx)++;
1756                 }
1757                 for (tc_index = 0;
1758                      tc_index < devlink_sb->egress_tc_count; tc_index++) {
1759                         if (*p_idx < start) {
1760                                 (*p_idx)++;
1761                                 continue;
1762                         }
1763                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
1764                                                               devlink_port,
1765                                                               devlink_sb,
1766                                                               tc_index,
1767                                                               DEVLINK_SB_POOL_TYPE_EGRESS,
1768                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
1769                                                               portid, seq,
1770                                                               NLM_F_MULTI);
1771                         if (err)
1772                                 return err;
1773                         (*p_idx)++;
1774                 }
1775         }
1776         return 0;
1777 }
1778
1779 static int
1780 devlink_nl_cmd_sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
1781                                           struct netlink_callback *cb)
1782 {
1783         struct devlink *devlink;
1784         struct devlink_sb *devlink_sb;
1785         int start = cb->args[0];
1786         int idx = 0;
1787         int err = 0;
1788
1789         mutex_lock(&devlink_mutex);
1790         list_for_each_entry(devlink, &devlink_list, list) {
1791                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
1792                     !devlink->ops->sb_tc_pool_bind_get)
1793                         continue;
1794
1795                 mutex_lock(&devlink->lock);
1796                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
1797                         err = __sb_tc_pool_bind_get_dumpit(msg, start, &idx,
1798                                                            devlink,
1799                                                            devlink_sb,
1800                                                            NETLINK_CB(cb->skb).portid,
1801                                                            cb->nlh->nlmsg_seq);
1802                         if (err == -EOPNOTSUPP) {
1803                                 err = 0;
1804                         } else if (err) {
1805                                 mutex_unlock(&devlink->lock);
1806                                 goto out;
1807                         }
1808                 }
1809                 mutex_unlock(&devlink->lock);
1810         }
1811 out:
1812         mutex_unlock(&devlink_mutex);
1813
1814         if (err != -EMSGSIZE)
1815                 return err;
1816
1817         cb->args[0] = idx;
1818         return msg->len;
1819 }
1820
1821 static int devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
1822                                        unsigned int sb_index, u16 tc_index,
1823                                        enum devlink_sb_pool_type pool_type,
1824                                        u16 pool_index, u32 threshold,
1825                                        struct netlink_ext_ack *extack)
1826
1827 {
1828         const struct devlink_ops *ops = devlink_port->devlink->ops;
1829
1830         if (ops->sb_tc_pool_bind_set)
1831                 return ops->sb_tc_pool_bind_set(devlink_port, sb_index,
1832                                                 tc_index, pool_type,
1833                                                 pool_index, threshold, extack);
1834         return -EOPNOTSUPP;
1835 }
1836
1837 static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
1838                                                    struct genl_info *info)
1839 {
1840         struct devlink_port *devlink_port = info->user_ptr[1];
1841         struct devlink *devlink = info->user_ptr[0];
1842         enum devlink_sb_pool_type pool_type;
1843         struct devlink_sb *devlink_sb;
1844         u16 tc_index;
1845         u16 pool_index;
1846         u32 threshold;
1847         int err;
1848
1849         devlink_sb = devlink_sb_get_from_info(devlink, info);
1850         if (IS_ERR(devlink_sb))
1851                 return PTR_ERR(devlink_sb);
1852
1853         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
1854         if (err)
1855                 return err;
1856
1857         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
1858                                                 pool_type, &tc_index);
1859         if (err)
1860                 return err;
1861
1862         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
1863                                                   &pool_index);
1864         if (err)
1865                 return err;
1866
1867         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
1868                 return -EINVAL;
1869
1870         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
1871         return devlink_sb_tc_pool_bind_set(devlink_port, devlink_sb->index,
1872                                            tc_index, pool_type,
1873                                            pool_index, threshold, info->extack);
1874 }
1875
1876 static int devlink_nl_cmd_sb_occ_snapshot_doit(struct sk_buff *skb,
1877                                                struct genl_info *info)
1878 {
1879         struct devlink *devlink = info->user_ptr[0];
1880         const struct devlink_ops *ops = devlink->ops;
1881         struct devlink_sb *devlink_sb;
1882
1883         devlink_sb = devlink_sb_get_from_info(devlink, info);
1884         if (IS_ERR(devlink_sb))
1885                 return PTR_ERR(devlink_sb);
1886
1887         if (ops->sb_occ_snapshot)
1888                 return ops->sb_occ_snapshot(devlink, devlink_sb->index);
1889         return -EOPNOTSUPP;
1890 }
1891
1892 static int devlink_nl_cmd_sb_occ_max_clear_doit(struct sk_buff *skb,
1893                                                 struct genl_info *info)
1894 {
1895         struct devlink *devlink = info->user_ptr[0];
1896         const struct devlink_ops *ops = devlink->ops;
1897         struct devlink_sb *devlink_sb;
1898
1899         devlink_sb = devlink_sb_get_from_info(devlink, info);
1900         if (IS_ERR(devlink_sb))
1901                 return PTR_ERR(devlink_sb);
1902
1903         if (ops->sb_occ_max_clear)
1904                 return ops->sb_occ_max_clear(devlink, devlink_sb->index);
1905         return -EOPNOTSUPP;
1906 }
1907
1908 static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
1909                                    enum devlink_command cmd, u32 portid,
1910                                    u32 seq, int flags)
1911 {
1912         const struct devlink_ops *ops = devlink->ops;
1913         enum devlink_eswitch_encap_mode encap_mode;
1914         u8 inline_mode;
1915         void *hdr;
1916         int err = 0;
1917         u16 mode;
1918
1919         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1920         if (!hdr)
1921                 return -EMSGSIZE;
1922
1923         err = devlink_nl_put_handle(msg, devlink);
1924         if (err)
1925                 goto nla_put_failure;
1926
1927         if (ops->eswitch_mode_get) {
1928                 err = ops->eswitch_mode_get(devlink, &mode);
1929                 if (err)
1930                         goto nla_put_failure;
1931                 err = nla_put_u16(msg, DEVLINK_ATTR_ESWITCH_MODE, mode);
1932                 if (err)
1933                         goto nla_put_failure;
1934         }
1935
1936         if (ops->eswitch_inline_mode_get) {
1937                 err = ops->eswitch_inline_mode_get(devlink, &inline_mode);
1938                 if (err)
1939                         goto nla_put_failure;
1940                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_INLINE_MODE,
1941                                  inline_mode);
1942                 if (err)
1943                         goto nla_put_failure;
1944         }
1945
1946         if (ops->eswitch_encap_mode_get) {
1947                 err = ops->eswitch_encap_mode_get(devlink, &encap_mode);
1948                 if (err)
1949                         goto nla_put_failure;
1950                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_ENCAP_MODE, encap_mode);
1951                 if (err)
1952                         goto nla_put_failure;
1953         }
1954
1955         genlmsg_end(msg, hdr);
1956         return 0;
1957
1958 nla_put_failure:
1959         genlmsg_cancel(msg, hdr);
1960         return err;
1961 }
1962
1963 static int devlink_nl_cmd_eswitch_get_doit(struct sk_buff *skb,
1964                                            struct genl_info *info)
1965 {
1966         struct devlink *devlink = info->user_ptr[0];
1967         struct sk_buff *msg;
1968         int err;
1969
1970         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1971         if (!msg)
1972                 return -ENOMEM;
1973
1974         err = devlink_nl_eswitch_fill(msg, devlink, DEVLINK_CMD_ESWITCH_GET,
1975                                       info->snd_portid, info->snd_seq, 0);
1976
1977         if (err) {
1978                 nlmsg_free(msg);
1979                 return err;
1980         }
1981
1982         return genlmsg_reply(msg, info);
1983 }
1984
1985 static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb,
1986                                            struct genl_info *info)
1987 {
1988         struct devlink *devlink = info->user_ptr[0];
1989         const struct devlink_ops *ops = devlink->ops;
1990         enum devlink_eswitch_encap_mode encap_mode;
1991         u8 inline_mode;
1992         int err = 0;
1993         u16 mode;
1994
1995         if (info->attrs[DEVLINK_ATTR_ESWITCH_MODE]) {
1996                 if (!ops->eswitch_mode_set)
1997                         return -EOPNOTSUPP;
1998                 mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]);
1999                 err = ops->eswitch_mode_set(devlink, mode, info->extack);
2000                 if (err)
2001                         return err;
2002         }
2003
2004         if (info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]) {
2005                 if (!ops->eswitch_inline_mode_set)
2006                         return -EOPNOTSUPP;
2007                 inline_mode = nla_get_u8(
2008                                 info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]);
2009                 err = ops->eswitch_inline_mode_set(devlink, inline_mode,
2010                                                    info->extack);
2011                 if (err)
2012                         return err;
2013         }
2014
2015         if (info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]) {
2016                 if (!ops->eswitch_encap_mode_set)
2017                         return -EOPNOTSUPP;
2018                 encap_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]);
2019                 err = ops->eswitch_encap_mode_set(devlink, encap_mode,
2020                                                   info->extack);
2021                 if (err)
2022                         return err;
2023         }
2024
2025         return 0;
2026 }
2027
2028 int devlink_dpipe_match_put(struct sk_buff *skb,
2029                             struct devlink_dpipe_match *match)
2030 {
2031         struct devlink_dpipe_header *header = match->header;
2032         struct devlink_dpipe_field *field = &header->fields[match->field_id];
2033         struct nlattr *match_attr;
2034
2035         match_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_MATCH);
2036         if (!match_attr)
2037                 return -EMSGSIZE;
2038
2039         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_MATCH_TYPE, match->type) ||
2040             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, match->header_index) ||
2041             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
2042             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
2043             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
2044                 goto nla_put_failure;
2045
2046         nla_nest_end(skb, match_attr);
2047         return 0;
2048
2049 nla_put_failure:
2050         nla_nest_cancel(skb, match_attr);
2051         return -EMSGSIZE;
2052 }
2053 EXPORT_SYMBOL_GPL(devlink_dpipe_match_put);
2054
2055 static int devlink_dpipe_matches_put(struct devlink_dpipe_table *table,
2056                                      struct sk_buff *skb)
2057 {
2058         struct nlattr *matches_attr;
2059
2060         matches_attr = nla_nest_start_noflag(skb,
2061                                              DEVLINK_ATTR_DPIPE_TABLE_MATCHES);
2062         if (!matches_attr)
2063                 return -EMSGSIZE;
2064
2065         if (table->table_ops->matches_dump(table->priv, skb))
2066                 goto nla_put_failure;
2067
2068         nla_nest_end(skb, matches_attr);
2069         return 0;
2070
2071 nla_put_failure:
2072         nla_nest_cancel(skb, matches_attr);
2073         return -EMSGSIZE;
2074 }
2075
2076 int devlink_dpipe_action_put(struct sk_buff *skb,
2077                              struct devlink_dpipe_action *action)
2078 {
2079         struct devlink_dpipe_header *header = action->header;
2080         struct devlink_dpipe_field *field = &header->fields[action->field_id];
2081         struct nlattr *action_attr;
2082
2083         action_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ACTION);
2084         if (!action_attr)
2085                 return -EMSGSIZE;
2086
2087         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_ACTION_TYPE, action->type) ||
2088             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, action->header_index) ||
2089             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
2090             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
2091             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
2092                 goto nla_put_failure;
2093
2094         nla_nest_end(skb, action_attr);
2095         return 0;
2096
2097 nla_put_failure:
2098         nla_nest_cancel(skb, action_attr);
2099         return -EMSGSIZE;
2100 }
2101 EXPORT_SYMBOL_GPL(devlink_dpipe_action_put);
2102
2103 static int devlink_dpipe_actions_put(struct devlink_dpipe_table *table,
2104                                      struct sk_buff *skb)
2105 {
2106         struct nlattr *actions_attr;
2107
2108         actions_attr = nla_nest_start_noflag(skb,
2109                                              DEVLINK_ATTR_DPIPE_TABLE_ACTIONS);
2110         if (!actions_attr)
2111                 return -EMSGSIZE;
2112
2113         if (table->table_ops->actions_dump(table->priv, skb))
2114                 goto nla_put_failure;
2115
2116         nla_nest_end(skb, actions_attr);
2117         return 0;
2118
2119 nla_put_failure:
2120         nla_nest_cancel(skb, actions_attr);
2121         return -EMSGSIZE;
2122 }
2123
2124 static int devlink_dpipe_table_put(struct sk_buff *skb,
2125                                    struct devlink_dpipe_table *table)
2126 {
2127         struct nlattr *table_attr;
2128         u64 table_size;
2129
2130         table_size = table->table_ops->size_get(table->priv);
2131         table_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLE);
2132         if (!table_attr)
2133                 return -EMSGSIZE;
2134
2135         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_TABLE_NAME, table->name) ||
2136             nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_SIZE, table_size,
2137                               DEVLINK_ATTR_PAD))
2138                 goto nla_put_failure;
2139         if (nla_put_u8(skb, DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED,
2140                        table->counters_enabled))
2141                 goto nla_put_failure;
2142
2143         if (table->resource_valid) {
2144                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_ID,
2145                                       table->resource_id, DEVLINK_ATTR_PAD) ||
2146                     nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_UNITS,
2147                                       table->resource_units, DEVLINK_ATTR_PAD))
2148                         goto nla_put_failure;
2149         }
2150         if (devlink_dpipe_matches_put(table, skb))
2151                 goto nla_put_failure;
2152
2153         if (devlink_dpipe_actions_put(table, skb))
2154                 goto nla_put_failure;
2155
2156         nla_nest_end(skb, table_attr);
2157         return 0;
2158
2159 nla_put_failure:
2160         nla_nest_cancel(skb, table_attr);
2161         return -EMSGSIZE;
2162 }
2163
2164 static int devlink_dpipe_send_and_alloc_skb(struct sk_buff **pskb,
2165                                             struct genl_info *info)
2166 {
2167         int err;
2168
2169         if (*pskb) {
2170                 err = genlmsg_reply(*pskb, info);
2171                 if (err)
2172                         return err;
2173         }
2174         *pskb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
2175         if (!*pskb)
2176                 return -ENOMEM;
2177         return 0;
2178 }
2179
2180 static int devlink_dpipe_tables_fill(struct genl_info *info,
2181                                      enum devlink_command cmd, int flags,
2182                                      struct list_head *dpipe_tables,
2183                                      const char *table_name)
2184 {
2185         struct devlink *devlink = info->user_ptr[0];
2186         struct devlink_dpipe_table *table;
2187         struct nlattr *tables_attr;
2188         struct sk_buff *skb = NULL;
2189         struct nlmsghdr *nlh;
2190         bool incomplete;
2191         void *hdr;
2192         int i;
2193         int err;
2194
2195         table = list_first_entry(dpipe_tables,
2196                                  struct devlink_dpipe_table, list);
2197 start_again:
2198         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
2199         if (err)
2200                 return err;
2201
2202         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
2203                           &devlink_nl_family, NLM_F_MULTI, cmd);
2204         if (!hdr) {
2205                 nlmsg_free(skb);
2206                 return -EMSGSIZE;
2207         }
2208
2209         if (devlink_nl_put_handle(skb, devlink))
2210                 goto nla_put_failure;
2211         tables_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLES);
2212         if (!tables_attr)
2213                 goto nla_put_failure;
2214
2215         i = 0;
2216         incomplete = false;
2217         list_for_each_entry_from(table, dpipe_tables, list) {
2218                 if (!table_name) {
2219                         err = devlink_dpipe_table_put(skb, table);
2220                         if (err) {
2221                                 if (!i)
2222                                         goto err_table_put;
2223                                 incomplete = true;
2224                                 break;
2225                         }
2226                 } else {
2227                         if (!strcmp(table->name, table_name)) {
2228                                 err = devlink_dpipe_table_put(skb, table);
2229                                 if (err)
2230                                         break;
2231                         }
2232                 }
2233                 i++;
2234         }
2235
2236         nla_nest_end(skb, tables_attr);
2237         genlmsg_end(skb, hdr);
2238         if (incomplete)
2239                 goto start_again;
2240
2241 send_done:
2242         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
2243                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
2244         if (!nlh) {
2245                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
2246                 if (err)
2247                         return err;
2248                 goto send_done;
2249         }
2250
2251         return genlmsg_reply(skb, info);
2252
2253 nla_put_failure:
2254         err = -EMSGSIZE;
2255 err_table_put:
2256         nlmsg_free(skb);
2257         return err;
2258 }
2259
2260 static int devlink_nl_cmd_dpipe_table_get(struct sk_buff *skb,
2261                                           struct genl_info *info)
2262 {
2263         struct devlink *devlink = info->user_ptr[0];
2264         const char *table_name =  NULL;
2265
2266         if (info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
2267                 table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
2268
2269         return devlink_dpipe_tables_fill(info, DEVLINK_CMD_DPIPE_TABLE_GET, 0,
2270                                          &devlink->dpipe_table_list,
2271                                          table_name);
2272 }
2273
2274 static int devlink_dpipe_value_put(struct sk_buff *skb,
2275                                    struct devlink_dpipe_value *value)
2276 {
2277         if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE,
2278                     value->value_size, value->value))
2279                 return -EMSGSIZE;
2280         if (value->mask)
2281                 if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE_MASK,
2282                             value->value_size, value->mask))
2283                         return -EMSGSIZE;
2284         if (value->mapping_valid)
2285                 if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_VALUE_MAPPING,
2286                                 value->mapping_value))
2287                         return -EMSGSIZE;
2288         return 0;
2289 }
2290
2291 static int devlink_dpipe_action_value_put(struct sk_buff *skb,
2292                                           struct devlink_dpipe_value *value)
2293 {
2294         if (!value->action)
2295                 return -EINVAL;
2296         if (devlink_dpipe_action_put(skb, value->action))
2297                 return -EMSGSIZE;
2298         if (devlink_dpipe_value_put(skb, value))
2299                 return -EMSGSIZE;
2300         return 0;
2301 }
2302
2303 static int devlink_dpipe_action_values_put(struct sk_buff *skb,
2304                                            struct devlink_dpipe_value *values,
2305                                            unsigned int values_count)
2306 {
2307         struct nlattr *action_attr;
2308         int i;
2309         int err;
2310
2311         for (i = 0; i < values_count; i++) {
2312                 action_attr = nla_nest_start_noflag(skb,
2313                                                     DEVLINK_ATTR_DPIPE_ACTION_VALUE);
2314                 if (!action_attr)
2315                         return -EMSGSIZE;
2316                 err = devlink_dpipe_action_value_put(skb, &values[i]);
2317                 if (err)
2318                         goto err_action_value_put;
2319                 nla_nest_end(skb, action_attr);
2320         }
2321         return 0;
2322
2323 err_action_value_put:
2324         nla_nest_cancel(skb, action_attr);
2325         return err;
2326 }
2327
2328 static int devlink_dpipe_match_value_put(struct sk_buff *skb,
2329                                          struct devlink_dpipe_value *value)
2330 {
2331         if (!value->match)
2332                 return -EINVAL;
2333         if (devlink_dpipe_match_put(skb, value->match))
2334                 return -EMSGSIZE;
2335         if (devlink_dpipe_value_put(skb, value))
2336                 return -EMSGSIZE;
2337         return 0;
2338 }
2339
2340 static int devlink_dpipe_match_values_put(struct sk_buff *skb,
2341                                           struct devlink_dpipe_value *values,
2342                                           unsigned int values_count)
2343 {
2344         struct nlattr *match_attr;
2345         int i;
2346         int err;
2347
2348         for (i = 0; i < values_count; i++) {
2349                 match_attr = nla_nest_start_noflag(skb,
2350                                                    DEVLINK_ATTR_DPIPE_MATCH_VALUE);
2351                 if (!match_attr)
2352                         return -EMSGSIZE;
2353                 err = devlink_dpipe_match_value_put(skb, &values[i]);
2354                 if (err)
2355                         goto err_match_value_put;
2356                 nla_nest_end(skb, match_attr);
2357         }
2358         return 0;
2359
2360 err_match_value_put:
2361         nla_nest_cancel(skb, match_attr);
2362         return err;
2363 }
2364
2365 static int devlink_dpipe_entry_put(struct sk_buff *skb,
2366                                    struct devlink_dpipe_entry *entry)
2367 {
2368         struct nlattr *entry_attr, *matches_attr, *actions_attr;
2369         int err;
2370
2371         entry_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ENTRY);
2372         if (!entry_attr)
2373                 return  -EMSGSIZE;
2374
2375         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_INDEX, entry->index,
2376                               DEVLINK_ATTR_PAD))
2377                 goto nla_put_failure;
2378         if (entry->counter_valid)
2379                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_COUNTER,
2380                                       entry->counter, DEVLINK_ATTR_PAD))
2381                         goto nla_put_failure;
2382
2383         matches_attr = nla_nest_start_noflag(skb,
2384                                              DEVLINK_ATTR_DPIPE_ENTRY_MATCH_VALUES);
2385         if (!matches_attr)
2386                 goto nla_put_failure;
2387
2388         err = devlink_dpipe_match_values_put(skb, entry->match_values,
2389                                              entry->match_values_count);
2390         if (err) {
2391                 nla_nest_cancel(skb, matches_attr);
2392                 goto err_match_values_put;
2393         }
2394         nla_nest_end(skb, matches_attr);
2395
2396         actions_attr = nla_nest_start_noflag(skb,
2397                                              DEVLINK_ATTR_DPIPE_ENTRY_ACTION_VALUES);
2398         if (!actions_attr)
2399                 goto nla_put_failure;
2400
2401         err = devlink_dpipe_action_values_put(skb, entry->action_values,
2402                                               entry->action_values_count);
2403         if (err) {
2404                 nla_nest_cancel(skb, actions_attr);
2405                 goto err_action_values_put;
2406         }
2407         nla_nest_end(skb, actions_attr);
2408
2409         nla_nest_end(skb, entry_attr);
2410         return 0;
2411
2412 nla_put_failure:
2413         err = -EMSGSIZE;
2414 err_match_values_put:
2415 err_action_values_put:
2416         nla_nest_cancel(skb, entry_attr);
2417         return err;
2418 }
2419
2420 static struct devlink_dpipe_table *
2421 devlink_dpipe_table_find(struct list_head *dpipe_tables,
2422                          const char *table_name, struct devlink *devlink)
2423 {
2424         struct devlink_dpipe_table *table;
2425         list_for_each_entry_rcu(table, dpipe_tables, list,
2426                                 lockdep_is_held(&devlink->lock)) {
2427                 if (!strcmp(table->name, table_name))
2428                         return table;
2429         }
2430         return NULL;
2431 }
2432
2433 int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx)
2434 {
2435         struct devlink *devlink;
2436         int err;
2437
2438         err = devlink_dpipe_send_and_alloc_skb(&dump_ctx->skb,
2439                                                dump_ctx->info);
2440         if (err)
2441                 return err;
2442
2443         dump_ctx->hdr = genlmsg_put(dump_ctx->skb,
2444                                     dump_ctx->info->snd_portid,
2445                                     dump_ctx->info->snd_seq,
2446                                     &devlink_nl_family, NLM_F_MULTI,
2447                                     dump_ctx->cmd);
2448         if (!dump_ctx->hdr)
2449                 goto nla_put_failure;
2450
2451         devlink = dump_ctx->info->user_ptr[0];
2452         if (devlink_nl_put_handle(dump_ctx->skb, devlink))
2453                 goto nla_put_failure;
2454         dump_ctx->nest = nla_nest_start_noflag(dump_ctx->skb,
2455                                                DEVLINK_ATTR_DPIPE_ENTRIES);
2456         if (!dump_ctx->nest)
2457                 goto nla_put_failure;
2458         return 0;
2459
2460 nla_put_failure:
2461         nlmsg_free(dump_ctx->skb);
2462         return -EMSGSIZE;
2463 }
2464 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_prepare);
2465
2466 int devlink_dpipe_entry_ctx_append(struct devlink_dpipe_dump_ctx *dump_ctx,
2467                                    struct devlink_dpipe_entry *entry)
2468 {
2469         return devlink_dpipe_entry_put(dump_ctx->skb, entry);
2470 }
2471 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_append);
2472
2473 int devlink_dpipe_entry_ctx_close(struct devlink_dpipe_dump_ctx *dump_ctx)
2474 {
2475         nla_nest_end(dump_ctx->skb, dump_ctx->nest);
2476         genlmsg_end(dump_ctx->skb, dump_ctx->hdr);
2477         return 0;
2478 }
2479 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_close);
2480
2481 void devlink_dpipe_entry_clear(struct devlink_dpipe_entry *entry)
2482
2483 {
2484         unsigned int value_count, value_index;
2485         struct devlink_dpipe_value *value;
2486
2487         value = entry->action_values;
2488         value_count = entry->action_values_count;
2489         for (value_index = 0; value_index < value_count; value_index++) {
2490                 kfree(value[value_index].value);
2491                 kfree(value[value_index].mask);
2492         }
2493
2494         value = entry->match_values;
2495         value_count = entry->match_values_count;
2496         for (value_index = 0; value_index < value_count; value_index++) {
2497                 kfree(value[value_index].value);
2498                 kfree(value[value_index].mask);
2499         }
2500 }
2501 EXPORT_SYMBOL(devlink_dpipe_entry_clear);
2502
2503 static int devlink_dpipe_entries_fill(struct genl_info *info,
2504                                       enum devlink_command cmd, int flags,
2505                                       struct devlink_dpipe_table *table)
2506 {
2507         struct devlink_dpipe_dump_ctx dump_ctx;
2508         struct nlmsghdr *nlh;
2509         int err;
2510
2511         dump_ctx.skb = NULL;
2512         dump_ctx.cmd = cmd;
2513         dump_ctx.info = info;
2514
2515         err = table->table_ops->entries_dump(table->priv,
2516                                              table->counters_enabled,
2517                                              &dump_ctx);
2518         if (err)
2519                 return err;
2520
2521 send_done:
2522         nlh = nlmsg_put(dump_ctx.skb, info->snd_portid, info->snd_seq,
2523                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
2524         if (!nlh) {
2525                 err = devlink_dpipe_send_and_alloc_skb(&dump_ctx.skb, info);
2526                 if (err)
2527                         return err;
2528                 goto send_done;
2529         }
2530         return genlmsg_reply(dump_ctx.skb, info);
2531 }
2532
2533 static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb,
2534                                             struct genl_info *info)
2535 {
2536         struct devlink *devlink = info->user_ptr[0];
2537         struct devlink_dpipe_table *table;
2538         const char *table_name;
2539
2540         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
2541                 return -EINVAL;
2542
2543         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
2544         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
2545                                          table_name, devlink);
2546         if (!table)
2547                 return -EINVAL;
2548
2549         if (!table->table_ops->entries_dump)
2550                 return -EINVAL;
2551
2552         return devlink_dpipe_entries_fill(info, DEVLINK_CMD_DPIPE_ENTRIES_GET,
2553                                           0, table);
2554 }
2555
2556 static int devlink_dpipe_fields_put(struct sk_buff *skb,
2557                                     const struct devlink_dpipe_header *header)
2558 {
2559         struct devlink_dpipe_field *field;
2560         struct nlattr *field_attr;
2561         int i;
2562
2563         for (i = 0; i < header->fields_count; i++) {
2564                 field = &header->fields[i];
2565                 field_attr = nla_nest_start_noflag(skb,
2566                                                    DEVLINK_ATTR_DPIPE_FIELD);
2567                 if (!field_attr)
2568                         return -EMSGSIZE;
2569                 if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_FIELD_NAME, field->name) ||
2570                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
2571                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_BITWIDTH, field->bitwidth) ||
2572                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE, field->mapping_type))
2573                         goto nla_put_failure;
2574                 nla_nest_end(skb, field_attr);
2575         }
2576         return 0;
2577
2578 nla_put_failure:
2579         nla_nest_cancel(skb, field_attr);
2580         return -EMSGSIZE;
2581 }
2582
2583 static int devlink_dpipe_header_put(struct sk_buff *skb,
2584                                     struct devlink_dpipe_header *header)
2585 {
2586         struct nlattr *fields_attr, *header_attr;
2587         int err;
2588
2589         header_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADER);
2590         if (!header_attr)
2591                 return -EMSGSIZE;
2592
2593         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_HEADER_NAME, header->name) ||
2594             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
2595             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
2596                 goto nla_put_failure;
2597
2598         fields_attr = nla_nest_start_noflag(skb,
2599                                             DEVLINK_ATTR_DPIPE_HEADER_FIELDS);
2600         if (!fields_attr)
2601                 goto nla_put_failure;
2602
2603         err = devlink_dpipe_fields_put(skb, header);
2604         if (err) {
2605                 nla_nest_cancel(skb, fields_attr);
2606                 goto nla_put_failure;
2607         }
2608         nla_nest_end(skb, fields_attr);
2609         nla_nest_end(skb, header_attr);
2610         return 0;
2611
2612 nla_put_failure:
2613         err = -EMSGSIZE;
2614         nla_nest_cancel(skb, header_attr);
2615         return err;
2616 }
2617
2618 static int devlink_dpipe_headers_fill(struct genl_info *info,
2619                                       enum devlink_command cmd, int flags,
2620                                       struct devlink_dpipe_headers *
2621                                       dpipe_headers)
2622 {
2623         struct devlink *devlink = info->user_ptr[0];
2624         struct nlattr *headers_attr;
2625         struct sk_buff *skb = NULL;
2626         struct nlmsghdr *nlh;
2627         void *hdr;
2628         int i, j;
2629         int err;
2630
2631         i = 0;
2632 start_again:
2633         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
2634         if (err)
2635                 return err;
2636
2637         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
2638                           &devlink_nl_family, NLM_F_MULTI, cmd);
2639         if (!hdr) {
2640                 nlmsg_free(skb);
2641                 return -EMSGSIZE;
2642         }
2643
2644         if (devlink_nl_put_handle(skb, devlink))
2645                 goto nla_put_failure;
2646         headers_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADERS);
2647         if (!headers_attr)
2648                 goto nla_put_failure;
2649
2650         j = 0;
2651         for (; i < dpipe_headers->headers_count; i++) {
2652                 err = devlink_dpipe_header_put(skb, dpipe_headers->headers[i]);
2653                 if (err) {
2654                         if (!j)
2655                                 goto err_table_put;
2656                         break;
2657                 }
2658                 j++;
2659         }
2660         nla_nest_end(skb, headers_attr);
2661         genlmsg_end(skb, hdr);
2662         if (i != dpipe_headers->headers_count)
2663                 goto start_again;
2664
2665 send_done:
2666         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
2667                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
2668         if (!nlh) {
2669                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
2670                 if (err)
2671                         return err;
2672                 goto send_done;
2673         }
2674         return genlmsg_reply(skb, info);
2675
2676 nla_put_failure:
2677         err = -EMSGSIZE;
2678 err_table_put:
2679         nlmsg_free(skb);
2680         return err;
2681 }
2682
2683 static int devlink_nl_cmd_dpipe_headers_get(struct sk_buff *skb,
2684                                             struct genl_info *info)
2685 {
2686         struct devlink *devlink = info->user_ptr[0];
2687
2688         if (!devlink->dpipe_headers)
2689                 return -EOPNOTSUPP;
2690         return devlink_dpipe_headers_fill(info, DEVLINK_CMD_DPIPE_HEADERS_GET,
2691                                           0, devlink->dpipe_headers);
2692 }
2693
2694 static int devlink_dpipe_table_counters_set(struct devlink *devlink,
2695                                             const char *table_name,
2696                                             bool enable)
2697 {
2698         struct devlink_dpipe_table *table;
2699
2700         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
2701                                          table_name, devlink);
2702         if (!table)
2703                 return -EINVAL;
2704
2705         if (table->counter_control_extern)
2706                 return -EOPNOTSUPP;
2707
2708         if (!(table->counters_enabled ^ enable))
2709                 return 0;
2710
2711         table->counters_enabled = enable;
2712         if (table->table_ops->counters_set_update)
2713                 table->table_ops->counters_set_update(table->priv, enable);
2714         return 0;
2715 }
2716
2717 static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb,
2718                                                    struct genl_info *info)
2719 {
2720         struct devlink *devlink = info->user_ptr[0];
2721         const char *table_name;
2722         bool counters_enable;
2723
2724         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME] ||
2725             !info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED])
2726                 return -EINVAL;
2727
2728         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
2729         counters_enable = !!nla_get_u8(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED]);
2730
2731         return devlink_dpipe_table_counters_set(devlink, table_name,
2732                                                 counters_enable);
2733 }
2734
2735 static struct devlink_resource *
2736 devlink_resource_find(struct devlink *devlink,
2737                       struct devlink_resource *resource, u64 resource_id)
2738 {
2739         struct list_head *resource_list;
2740
2741         if (resource)
2742                 resource_list = &resource->resource_list;
2743         else
2744                 resource_list = &devlink->resource_list;
2745
2746         list_for_each_entry(resource, resource_list, list) {
2747                 struct devlink_resource *child_resource;
2748
2749                 if (resource->id == resource_id)
2750                         return resource;
2751
2752                 child_resource = devlink_resource_find(devlink, resource,
2753                                                        resource_id);
2754                 if (child_resource)
2755                         return child_resource;
2756         }
2757         return NULL;
2758 }
2759
2760 static void
2761 devlink_resource_validate_children(struct devlink_resource *resource)
2762 {
2763         struct devlink_resource *child_resource;
2764         bool size_valid = true;
2765         u64 parts_size = 0;
2766
2767         if (list_empty(&resource->resource_list))
2768                 goto out;
2769
2770         list_for_each_entry(child_resource, &resource->resource_list, list)
2771                 parts_size += child_resource->size_new;
2772
2773         if (parts_size > resource->size_new)
2774                 size_valid = false;
2775 out:
2776         resource->size_valid = size_valid;
2777 }
2778
2779 static int
2780 devlink_resource_validate_size(struct devlink_resource *resource, u64 size,
2781                                struct netlink_ext_ack *extack)
2782 {
2783         u64 reminder;
2784         int err = 0;
2785
2786         if (size > resource->size_params.size_max) {
2787                 NL_SET_ERR_MSG_MOD(extack, "Size larger than maximum");
2788                 err = -EINVAL;
2789         }
2790
2791         if (size < resource->size_params.size_min) {
2792                 NL_SET_ERR_MSG_MOD(extack, "Size smaller than minimum");
2793                 err = -EINVAL;
2794         }
2795
2796         div64_u64_rem(size, resource->size_params.size_granularity, &reminder);
2797         if (reminder) {
2798                 NL_SET_ERR_MSG_MOD(extack, "Wrong granularity");
2799                 err = -EINVAL;
2800         }
2801
2802         return err;
2803 }
2804
2805 static int devlink_nl_cmd_resource_set(struct sk_buff *skb,
2806                                        struct genl_info *info)
2807 {
2808         struct devlink *devlink = info->user_ptr[0];
2809         struct devlink_resource *resource;
2810         u64 resource_id;
2811         u64 size;
2812         int err;
2813
2814         if (!info->attrs[DEVLINK_ATTR_RESOURCE_ID] ||
2815             !info->attrs[DEVLINK_ATTR_RESOURCE_SIZE])
2816                 return -EINVAL;
2817         resource_id = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_ID]);
2818
2819         resource = devlink_resource_find(devlink, NULL, resource_id);
2820         if (!resource)
2821                 return -EINVAL;
2822
2823         size = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_SIZE]);
2824         err = devlink_resource_validate_size(resource, size, info->extack);
2825         if (err)
2826                 return err;
2827
2828         resource->size_new = size;
2829         devlink_resource_validate_children(resource);
2830         if (resource->parent)
2831                 devlink_resource_validate_children(resource->parent);
2832         return 0;
2833 }
2834
2835 static int
2836 devlink_resource_size_params_put(struct devlink_resource *resource,
2837                                  struct sk_buff *skb)
2838 {
2839         struct devlink_resource_size_params *size_params;
2840
2841         size_params = &resource->size_params;
2842         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_GRAN,
2843                               size_params->size_granularity, DEVLINK_ATTR_PAD) ||
2844             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MAX,
2845                               size_params->size_max, DEVLINK_ATTR_PAD) ||
2846             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MIN,
2847                               size_params->size_min, DEVLINK_ATTR_PAD) ||
2848             nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_UNIT, size_params->unit))
2849                 return -EMSGSIZE;
2850         return 0;
2851 }
2852
2853 static int devlink_resource_occ_put(struct devlink_resource *resource,
2854                                     struct sk_buff *skb)
2855 {
2856         if (!resource->occ_get)
2857                 return 0;
2858         return nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_OCC,
2859                                  resource->occ_get(resource->occ_get_priv),
2860                                  DEVLINK_ATTR_PAD);
2861 }
2862
2863 static int devlink_resource_put(struct devlink *devlink, struct sk_buff *skb,
2864                                 struct devlink_resource *resource)
2865 {
2866         struct devlink_resource *child_resource;
2867         struct nlattr *child_resource_attr;
2868         struct nlattr *resource_attr;
2869
2870         resource_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_RESOURCE);
2871         if (!resource_attr)
2872                 return -EMSGSIZE;
2873
2874         if (nla_put_string(skb, DEVLINK_ATTR_RESOURCE_NAME, resource->name) ||
2875             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE, resource->size,
2876                               DEVLINK_ATTR_PAD) ||
2877             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_ID, resource->id,
2878                               DEVLINK_ATTR_PAD))
2879                 goto nla_put_failure;
2880         if (resource->size != resource->size_new)
2881                 nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_NEW,
2882                                   resource->size_new, DEVLINK_ATTR_PAD);
2883         if (devlink_resource_occ_put(resource, skb))
2884                 goto nla_put_failure;
2885         if (devlink_resource_size_params_put(resource, skb))
2886                 goto nla_put_failure;
2887         if (list_empty(&resource->resource_list))
2888                 goto out;
2889
2890         if (nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_SIZE_VALID,
2891                        resource->size_valid))
2892                 goto nla_put_failure;
2893
2894         child_resource_attr = nla_nest_start_noflag(skb,
2895                                                     DEVLINK_ATTR_RESOURCE_LIST);
2896         if (!child_resource_attr)
2897                 goto nla_put_failure;
2898
2899         list_for_each_entry(child_resource, &resource->resource_list, list) {
2900                 if (devlink_resource_put(devlink, skb, child_resource))
2901                         goto resource_put_failure;
2902         }
2903
2904         nla_nest_end(skb, child_resource_attr);
2905 out:
2906         nla_nest_end(skb, resource_attr);
2907         return 0;
2908
2909 resource_put_failure:
2910         nla_nest_cancel(skb, child_resource_attr);
2911 nla_put_failure:
2912         nla_nest_cancel(skb, resource_attr);
2913         return -EMSGSIZE;
2914 }
2915
2916 static int devlink_resource_fill(struct genl_info *info,
2917                                  enum devlink_command cmd, int flags)
2918 {
2919         struct devlink *devlink = info->user_ptr[0];
2920         struct devlink_resource *resource;
2921         struct nlattr *resources_attr;
2922         struct sk_buff *skb = NULL;
2923         struct nlmsghdr *nlh;
2924         bool incomplete;
2925         void *hdr;
2926         int i;
2927         int err;
2928
2929         resource = list_first_entry(&devlink->resource_list,
2930                                     struct devlink_resource, list);
2931 start_again:
2932         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
2933         if (err)
2934                 return err;
2935
2936         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
2937                           &devlink_nl_family, NLM_F_MULTI, cmd);
2938         if (!hdr) {
2939                 nlmsg_free(skb);
2940                 return -EMSGSIZE;
2941         }
2942
2943         if (devlink_nl_put_handle(skb, devlink))
2944                 goto nla_put_failure;
2945
2946         resources_attr = nla_nest_start_noflag(skb,
2947                                                DEVLINK_ATTR_RESOURCE_LIST);
2948         if (!resources_attr)
2949                 goto nla_put_failure;
2950
2951         incomplete = false;
2952         i = 0;
2953         list_for_each_entry_from(resource, &devlink->resource_list, list) {
2954                 err = devlink_resource_put(devlink, skb, resource);
2955                 if (err) {
2956                         if (!i)
2957                                 goto err_resource_put;
2958                         incomplete = true;
2959                         break;
2960                 }
2961                 i++;
2962         }
2963         nla_nest_end(skb, resources_attr);
2964         genlmsg_end(skb, hdr);
2965         if (incomplete)
2966                 goto start_again;
2967 send_done:
2968         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
2969                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
2970         if (!nlh) {
2971                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
2972                 if (err)
2973                         return err;
2974                 goto send_done;
2975         }
2976         return genlmsg_reply(skb, info);
2977
2978 nla_put_failure:
2979         err = -EMSGSIZE;
2980 err_resource_put:
2981         nlmsg_free(skb);
2982         return err;
2983 }
2984
2985 static int devlink_nl_cmd_resource_dump(struct sk_buff *skb,
2986                                         struct genl_info *info)
2987 {
2988         struct devlink *devlink = info->user_ptr[0];
2989
2990         if (list_empty(&devlink->resource_list))
2991                 return -EOPNOTSUPP;
2992
2993         return devlink_resource_fill(info, DEVLINK_CMD_RESOURCE_DUMP, 0);
2994 }
2995
2996 static int
2997 devlink_resources_validate(struct devlink *devlink,
2998                            struct devlink_resource *resource,
2999                            struct genl_info *info)
3000 {
3001         struct list_head *resource_list;
3002         int err = 0;
3003
3004         if (resource)
3005                 resource_list = &resource->resource_list;
3006         else
3007                 resource_list = &devlink->resource_list;
3008
3009         list_for_each_entry(resource, resource_list, list) {
3010                 if (!resource->size_valid)
3011                         return -EINVAL;
3012                 err = devlink_resources_validate(devlink, resource, info);
3013                 if (err)
3014                         return err;
3015         }
3016         return err;
3017 }
3018
3019 static struct net *devlink_netns_get(struct sk_buff *skb,
3020                                      struct genl_info *info)
3021 {
3022         struct nlattr *netns_pid_attr = info->attrs[DEVLINK_ATTR_NETNS_PID];
3023         struct nlattr *netns_fd_attr = info->attrs[DEVLINK_ATTR_NETNS_FD];
3024         struct nlattr *netns_id_attr = info->attrs[DEVLINK_ATTR_NETNS_ID];
3025         struct net *net;
3026
3027         if (!!netns_pid_attr + !!netns_fd_attr + !!netns_id_attr > 1) {
3028                 NL_SET_ERR_MSG_MOD(info->extack, "multiple netns identifying attributes specified");
3029                 return ERR_PTR(-EINVAL);
3030         }
3031
3032         if (netns_pid_attr) {
3033                 net = get_net_ns_by_pid(nla_get_u32(netns_pid_attr));
3034         } else if (netns_fd_attr) {
3035                 net = get_net_ns_by_fd(nla_get_u32(netns_fd_attr));
3036         } else if (netns_id_attr) {
3037                 net = get_net_ns_by_id(sock_net(skb->sk),
3038                                        nla_get_u32(netns_id_attr));
3039                 if (!net)
3040                         net = ERR_PTR(-EINVAL);
3041         } else {
3042                 WARN_ON(1);
3043                 net = ERR_PTR(-EINVAL);
3044         }
3045         if (IS_ERR(net)) {
3046                 NL_SET_ERR_MSG_MOD(info->extack, "Unknown network namespace");
3047                 return ERR_PTR(-EINVAL);
3048         }
3049         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
3050                 put_net(net);
3051                 return ERR_PTR(-EPERM);
3052         }
3053         return net;
3054 }
3055
3056 static void devlink_param_notify(struct devlink *devlink,
3057                                  unsigned int port_index,
3058                                  struct devlink_param_item *param_item,
3059                                  enum devlink_command cmd);
3060
3061 static void devlink_reload_netns_change(struct devlink *devlink,
3062                                         struct net *dest_net)
3063 {
3064         struct devlink_param_item *param_item;
3065
3066         /* Userspace needs to be notified about devlink objects
3067          * removed from original and entering new network namespace.
3068          * The rest of the devlink objects are re-created during
3069          * reload process so the notifications are generated separatelly.
3070          */
3071
3072         list_for_each_entry(param_item, &devlink->param_list, list)
3073                 devlink_param_notify(devlink, 0, param_item,
3074                                      DEVLINK_CMD_PARAM_DEL);
3075         devlink_notify(devlink, DEVLINK_CMD_DEL);
3076
3077         __devlink_net_set(devlink, dest_net);
3078
3079         devlink_notify(devlink, DEVLINK_CMD_NEW);
3080         list_for_each_entry(param_item, &devlink->param_list, list)
3081                 devlink_param_notify(devlink, 0, param_item,
3082                                      DEVLINK_CMD_PARAM_NEW);
3083 }
3084
3085 static bool devlink_reload_supported(const struct devlink_ops *ops)
3086 {
3087         return ops->reload_down && ops->reload_up;
3088 }
3089
3090 static void devlink_reload_failed_set(struct devlink *devlink,
3091                                       bool reload_failed)
3092 {
3093         if (devlink->reload_failed == reload_failed)
3094                 return;
3095         devlink->reload_failed = reload_failed;
3096         devlink_notify(devlink, DEVLINK_CMD_NEW);
3097 }
3098
3099 bool devlink_is_reload_failed(const struct devlink *devlink)
3100 {
3101         return devlink->reload_failed;
3102 }
3103 EXPORT_SYMBOL_GPL(devlink_is_reload_failed);
3104
3105 static void
3106 __devlink_reload_stats_update(struct devlink *devlink, u32 *reload_stats,
3107                               enum devlink_reload_limit limit, u32 actions_performed)
3108 {
3109         unsigned long actions = actions_performed;
3110         int stat_idx;
3111         int action;
3112
3113         for_each_set_bit(action, &actions, __DEVLINK_RELOAD_ACTION_MAX) {
3114                 stat_idx = limit * __DEVLINK_RELOAD_ACTION_MAX + action;
3115                 reload_stats[stat_idx]++;
3116         }
3117         devlink_notify(devlink, DEVLINK_CMD_NEW);
3118 }
3119
3120 static void
3121 devlink_reload_stats_update(struct devlink *devlink, enum devlink_reload_limit limit,
3122                             u32 actions_performed)
3123 {
3124         __devlink_reload_stats_update(devlink, devlink->stats.reload_stats, limit,
3125                                       actions_performed);
3126 }
3127
3128 /**
3129  *      devlink_remote_reload_actions_performed - Update devlink on reload actions
3130  *        performed which are not a direct result of devlink reload call.
3131  *
3132  *      This should be called by a driver after performing reload actions in case it was not
3133  *      a result of devlink reload call. For example fw_activate was performed as a result
3134  *      of devlink reload triggered fw_activate on another host.
3135  *      The motivation for this function is to keep data on reload actions performed on this
3136  *      function whether it was done due to direct devlink reload call or not.
3137  *
3138  *      @devlink: devlink
3139  *      @limit: reload limit
3140  *      @actions_performed: bitmask of actions performed
3141  */
3142 void devlink_remote_reload_actions_performed(struct devlink *devlink,
3143                                              enum devlink_reload_limit limit,
3144                                              u32 actions_performed)
3145 {
3146         if (WARN_ON(!actions_performed ||
3147                     actions_performed & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
3148                     actions_performed >= BIT(__DEVLINK_RELOAD_ACTION_MAX) ||
3149                     limit > DEVLINK_RELOAD_LIMIT_MAX))
3150                 return;
3151
3152         __devlink_reload_stats_update(devlink, devlink->stats.remote_reload_stats, limit,
3153                                       actions_performed);
3154 }
3155 EXPORT_SYMBOL_GPL(devlink_remote_reload_actions_performed);
3156
3157 static int devlink_reload(struct devlink *devlink, struct net *dest_net,
3158                           enum devlink_reload_action action, enum devlink_reload_limit limit,
3159                           u32 *actions_performed, struct netlink_ext_ack *extack)
3160 {
3161         u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
3162         int err;
3163
3164         if (!devlink->reload_enabled)
3165                 return -EOPNOTSUPP;
3166
3167         memcpy(remote_reload_stats, devlink->stats.remote_reload_stats,
3168                sizeof(remote_reload_stats));
3169         err = devlink->ops->reload_down(devlink, !!dest_net, action, limit, extack);
3170         if (err)
3171                 return err;
3172
3173         if (dest_net && !net_eq(dest_net, devlink_net(devlink)))
3174                 devlink_reload_netns_change(devlink, dest_net);
3175
3176         err = devlink->ops->reload_up(devlink, action, limit, actions_performed, extack);
3177         devlink_reload_failed_set(devlink, !!err);
3178         if (err)
3179                 return err;
3180
3181         WARN_ON(!(*actions_performed & BIT(action)));
3182         /* Catch driver on updating the remote action within devlink reload */
3183         WARN_ON(memcmp(remote_reload_stats, devlink->stats.remote_reload_stats,
3184                        sizeof(remote_reload_stats)));
3185         devlink_reload_stats_update(devlink, limit, *actions_performed);
3186         return 0;
3187 }
3188
3189 static int
3190 devlink_nl_reload_actions_performed_snd(struct devlink *devlink, u32 actions_performed,
3191                                         enum devlink_command cmd, struct genl_info *info)
3192 {
3193         struct sk_buff *msg;
3194         void *hdr;
3195
3196         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
3197         if (!msg)
3198                 return -ENOMEM;
3199
3200         hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq, &devlink_nl_family, 0, cmd);
3201         if (!hdr)
3202                 goto free_msg;
3203
3204         if (devlink_nl_put_handle(msg, devlink))
3205                 goto nla_put_failure;
3206
3207         if (nla_put_bitfield32(msg, DEVLINK_ATTR_RELOAD_ACTIONS_PERFORMED, actions_performed,
3208                                actions_performed))
3209                 goto nla_put_failure;
3210         genlmsg_end(msg, hdr);
3211
3212         return genlmsg_reply(msg, info);
3213
3214 nla_put_failure:
3215         genlmsg_cancel(msg, hdr);
3216 free_msg:
3217         nlmsg_free(msg);
3218         return -EMSGSIZE;
3219 }
3220
3221 static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
3222 {
3223         struct devlink *devlink = info->user_ptr[0];
3224         enum devlink_reload_action action;
3225         enum devlink_reload_limit limit;
3226         struct net *dest_net = NULL;
3227         u32 actions_performed;
3228         int err;
3229
3230         if (!devlink_reload_supported(devlink->ops))
3231                 return -EOPNOTSUPP;
3232
3233         err = devlink_resources_validate(devlink, NULL, info);
3234         if (err) {
3235                 NL_SET_ERR_MSG_MOD(info->extack, "resources size validation failed");
3236                 return err;
3237         }
3238
3239         if (info->attrs[DEVLINK_ATTR_NETNS_PID] ||
3240             info->attrs[DEVLINK_ATTR_NETNS_FD] ||
3241             info->attrs[DEVLINK_ATTR_NETNS_ID]) {
3242                 dest_net = devlink_netns_get(skb, info);
3243                 if (IS_ERR(dest_net))
3244                         return PTR_ERR(dest_net);
3245         }
3246
3247         if (info->attrs[DEVLINK_ATTR_RELOAD_ACTION])
3248                 action = nla_get_u8(info->attrs[DEVLINK_ATTR_RELOAD_ACTION]);
3249         else
3250                 action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT;
3251
3252         if (!devlink_reload_action_is_supported(devlink, action)) {
3253                 NL_SET_ERR_MSG_MOD(info->extack,
3254                                    "Requested reload action is not supported by the driver");
3255                 return -EOPNOTSUPP;
3256         }
3257
3258         limit = DEVLINK_RELOAD_LIMIT_UNSPEC;
3259         if (info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]) {
3260                 struct nla_bitfield32 limits;
3261                 u32 limits_selected;
3262
3263                 limits = nla_get_bitfield32(info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]);
3264                 limits_selected = limits.value & limits.selector;
3265                 if (!limits_selected) {
3266                         NL_SET_ERR_MSG_MOD(info->extack, "Invalid limit selected");
3267                         return -EINVAL;
3268                 }
3269                 for (limit = 0 ; limit <= DEVLINK_RELOAD_LIMIT_MAX ; limit++)
3270                         if (limits_selected & BIT(limit))
3271                                 break;
3272                 /* UAPI enables multiselection, but currently it is not used */
3273                 if (limits_selected != BIT(limit)) {
3274                         NL_SET_ERR_MSG_MOD(info->extack,
3275                                            "Multiselection of limit is not supported");
3276                         return -EOPNOTSUPP;
3277                 }
3278                 if (!devlink_reload_limit_is_supported(devlink, limit)) {
3279                         NL_SET_ERR_MSG_MOD(info->extack,
3280                                            "Requested limit is not supported by the driver");
3281                         return -EOPNOTSUPP;
3282                 }
3283                 if (devlink_reload_combination_is_invalid(action, limit)) {
3284                         NL_SET_ERR_MSG_MOD(info->extack,
3285                                            "Requested limit is invalid for this action");
3286                         return -EINVAL;
3287                 }
3288         }
3289         err = devlink_reload(devlink, dest_net, action, limit, &actions_performed, info->extack);
3290
3291         if (dest_net)
3292                 put_net(dest_net);
3293
3294         if (err)
3295                 return err;
3296         /* For backward compatibility generate reply only if attributes used by user */
3297         if (!info->attrs[DEVLINK_ATTR_RELOAD_ACTION] && !info->attrs[DEVLINK_ATTR_RELOAD_LIMITS])
3298                 return 0;
3299
3300         return devlink_nl_reload_actions_performed_snd(devlink, actions_performed,
3301                                                        DEVLINK_CMD_RELOAD, info);
3302 }
3303
3304 static int devlink_nl_flash_update_fill(struct sk_buff *msg,
3305                                         struct devlink *devlink,
3306                                         enum devlink_command cmd,
3307                                         struct devlink_flash_notify *params)
3308 {
3309         void *hdr;
3310
3311         hdr = genlmsg_put(msg, 0, 0, &devlink_nl_family, 0, cmd);
3312         if (!hdr)
3313                 return -EMSGSIZE;
3314
3315         if (devlink_nl_put_handle(msg, devlink))
3316                 goto nla_put_failure;
3317
3318         if (cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS)
3319                 goto out;
3320
3321         if (params->status_msg &&
3322             nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_MSG,
3323                            params->status_msg))
3324                 goto nla_put_failure;
3325         if (params->component &&
3326             nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_COMPONENT,
3327                            params->component))
3328                 goto nla_put_failure;
3329         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_DONE,
3330                               params->done, DEVLINK_ATTR_PAD))
3331                 goto nla_put_failure;
3332         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TOTAL,
3333                               params->total, DEVLINK_ATTR_PAD))
3334                 goto nla_put_failure;
3335         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TIMEOUT,
3336                               params->timeout, DEVLINK_ATTR_PAD))
3337                 goto nla_put_failure;
3338
3339 out:
3340         genlmsg_end(msg, hdr);
3341         return 0;
3342
3343 nla_put_failure:
3344         genlmsg_cancel(msg, hdr);
3345         return -EMSGSIZE;
3346 }
3347
3348 static void __devlink_flash_update_notify(struct devlink *devlink,
3349                                           enum devlink_command cmd,
3350                                           struct devlink_flash_notify *params)
3351 {
3352         struct sk_buff *msg;
3353         int err;
3354
3355         WARN_ON(cmd != DEVLINK_CMD_FLASH_UPDATE &&
3356                 cmd != DEVLINK_CMD_FLASH_UPDATE_END &&
3357                 cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS);
3358
3359         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
3360         if (!msg)
3361                 return;
3362
3363         err = devlink_nl_flash_update_fill(msg, devlink, cmd, params);
3364         if (err)
3365                 goto out_free_msg;
3366
3367         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
3368                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
3369         return;
3370
3371 out_free_msg:
3372         nlmsg_free(msg);
3373 }
3374
3375 void devlink_flash_update_begin_notify(struct devlink *devlink)
3376 {
3377         struct devlink_flash_notify params = { 0 };
3378
3379         __devlink_flash_update_notify(devlink,
3380                                       DEVLINK_CMD_FLASH_UPDATE,
3381                                       &params);
3382 }
3383 EXPORT_SYMBOL_GPL(devlink_flash_update_begin_notify);
3384
3385 void devlink_flash_update_end_notify(struct devlink *devlink)
3386 {
3387         struct devlink_flash_notify params = { 0 };
3388
3389         __devlink_flash_update_notify(devlink,
3390                                       DEVLINK_CMD_FLASH_UPDATE_END,
3391                                       &params);
3392 }
3393 EXPORT_SYMBOL_GPL(devlink_flash_update_end_notify);
3394
3395 void devlink_flash_update_status_notify(struct devlink *devlink,
3396                                         const char *status_msg,
3397                                         const char *component,
3398                                         unsigned long done,
3399                                         unsigned long total)
3400 {
3401         struct devlink_flash_notify params = {
3402                 .status_msg = status_msg,
3403                 .component = component,
3404                 .done = done,
3405                 .total = total,
3406         };
3407
3408         __devlink_flash_update_notify(devlink,
3409                                       DEVLINK_CMD_FLASH_UPDATE_STATUS,
3410                                       &params);
3411 }
3412 EXPORT_SYMBOL_GPL(devlink_flash_update_status_notify);
3413
3414 void devlink_flash_update_timeout_notify(struct devlink *devlink,
3415                                          const char *status_msg,
3416                                          const char *component,
3417                                          unsigned long timeout)
3418 {
3419         struct devlink_flash_notify params = {
3420                 .status_msg = status_msg,
3421                 .component = component,
3422                 .timeout = timeout,
3423         };
3424
3425         __devlink_flash_update_notify(devlink,
3426                                       DEVLINK_CMD_FLASH_UPDATE_STATUS,
3427                                       &params);
3428 }
3429 EXPORT_SYMBOL_GPL(devlink_flash_update_timeout_notify);
3430
3431 static int devlink_nl_cmd_flash_update(struct sk_buff *skb,
3432                                        struct genl_info *info)
3433 {
3434         struct nlattr *nla_component, *nla_overwrite_mask;
3435         struct devlink_flash_update_params params = {};
3436         struct devlink *devlink = info->user_ptr[0];
3437         u32 supported_params;
3438
3439         if (!devlink->ops->flash_update)
3440                 return -EOPNOTSUPP;
3441
3442         if (!info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME])
3443                 return -EINVAL;
3444
3445         supported_params = devlink->ops->supported_flash_update_params;
3446
3447         params.file_name = nla_data(info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME]);
3448
3449         nla_component = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT];
3450         if (nla_component) {
3451                 if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT)) {
3452                         NL_SET_ERR_MSG_ATTR(info->extack, nla_component,
3453                                             "component update is not supported by this device");
3454                         return -EOPNOTSUPP;
3455                 }
3456                 params.component = nla_data(nla_component);
3457         }
3458
3459         nla_overwrite_mask = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK];
3460         if (nla_overwrite_mask) {
3461                 struct nla_bitfield32 sections;
3462
3463                 if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK)) {
3464                         NL_SET_ERR_MSG_ATTR(info->extack, nla_overwrite_mask,
3465                                             "overwrite settings are not supported by this device");
3466                         return -EOPNOTSUPP;
3467                 }
3468                 sections = nla_get_bitfield32(nla_overwrite_mask);
3469                 params.overwrite_mask = sections.value & sections.selector;
3470         }
3471
3472         return devlink->ops->flash_update(devlink, &params, info->extack);
3473 }
3474
3475 static const struct devlink_param devlink_param_generic[] = {
3476         {
3477                 .id = DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
3478                 .name = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_NAME,
3479                 .type = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_TYPE,
3480         },
3481         {
3482                 .id = DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
3483                 .name = DEVLINK_PARAM_GENERIC_MAX_MACS_NAME,
3484                 .type = DEVLINK_PARAM_GENERIC_MAX_MACS_TYPE,
3485         },
3486         {
3487                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_SRIOV,
3488                 .name = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_NAME,
3489                 .type = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_TYPE,
3490         },
3491         {
3492                 .id = DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
3493                 .name = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_NAME,
3494                 .type = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_TYPE,
3495         },
3496         {
3497                 .id = DEVLINK_PARAM_GENERIC_ID_IGNORE_ARI,
3498                 .name = DEVLINK_PARAM_GENERIC_IGNORE_ARI_NAME,
3499                 .type = DEVLINK_PARAM_GENERIC_IGNORE_ARI_TYPE,
3500         },
3501         {
3502                 .id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MAX,
3503                 .name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_NAME,
3504                 .type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_TYPE,
3505         },
3506         {
3507                 .id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MIN,
3508                 .name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_NAME,
3509                 .type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_TYPE,
3510         },
3511         {
3512                 .id = DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,
3513                 .name = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_NAME,
3514                 .type = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_TYPE,
3515         },
3516         {
3517                 .id = DEVLINK_PARAM_GENERIC_ID_RESET_DEV_ON_DRV_PROBE,
3518                 .name = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_NAME,
3519                 .type = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_TYPE,
3520         },
3521         {
3522                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
3523                 .name = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_NAME,
3524                 .type = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_TYPE,
3525         },
3526         {
3527                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_REMOTE_DEV_RESET,
3528                 .name = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_NAME,
3529                 .type = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_TYPE,
3530         },
3531 };
3532
3533 static int devlink_param_generic_verify(const struct devlink_param *param)
3534 {
3535         /* verify it match generic parameter by id and name */
3536         if (param->id > DEVLINK_PARAM_GENERIC_ID_MAX)
3537                 return -EINVAL;
3538         if (strcmp(param->name, devlink_param_generic[param->id].name))
3539                 return -ENOENT;
3540
3541         WARN_ON(param->type != devlink_param_generic[param->id].type);
3542
3543         return 0;
3544 }
3545
3546 static int devlink_param_driver_verify(const struct devlink_param *param)
3547 {
3548         int i;
3549
3550         if (param->id <= DEVLINK_PARAM_GENERIC_ID_MAX)
3551                 return -EINVAL;
3552         /* verify no such name in generic params */
3553         for (i = 0; i <= DEVLINK_PARAM_GENERIC_ID_MAX; i++)
3554                 if (!strcmp(param->name, devlink_param_generic[i].name))
3555                         return -EEXIST;
3556
3557         return 0;
3558 }
3559
3560 static struct devlink_param_item *
3561 devlink_param_find_by_name(struct list_head *param_list,
3562                            const char *param_name)
3563 {
3564         struct devlink_param_item *param_item;
3565
3566         list_for_each_entry(param_item, param_list, list)
3567                 if (!strcmp(param_item->param->name, param_name))
3568                         return param_item;
3569         return NULL;
3570 }
3571
3572 static struct devlink_param_item *
3573 devlink_param_find_by_id(struct list_head *param_list, u32 param_id)
3574 {
3575         struct devlink_param_item *param_item;
3576
3577         list_for_each_entry(param_item, param_list, list)
3578                 if (param_item->param->id == param_id)
3579                         return param_item;
3580         return NULL;
3581 }
3582
3583 static bool
3584 devlink_param_cmode_is_supported(const struct devlink_param *param,
3585                                  enum devlink_param_cmode cmode)
3586 {
3587         return test_bit(cmode, &param->supported_cmodes);
3588 }
3589
3590 static int devlink_param_get(struct devlink *devlink,
3591                              const struct devlink_param *param,
3592                              struct devlink_param_gset_ctx *ctx)
3593 {
3594         if (!param->get)
3595                 return -EOPNOTSUPP;
3596         return param->get(devlink, param->id, ctx);
3597 }
3598
3599 static int devlink_param_set(struct devlink *devlink,
3600                              const struct devlink_param *param,
3601                              struct devlink_param_gset_ctx *ctx)
3602 {
3603         if (!param->set)
3604                 return -EOPNOTSUPP;
3605         return param->set(devlink, param->id, ctx);
3606 }
3607
3608 static int
3609 devlink_param_type_to_nla_type(enum devlink_param_type param_type)
3610 {
3611         switch (param_type) {
3612         case DEVLINK_PARAM_TYPE_U8:
3613                 return NLA_U8;
3614         case DEVLINK_PARAM_TYPE_U16:
3615                 return NLA_U16;
3616         case DEVLINK_PARAM_TYPE_U32:
3617                 return NLA_U32;
3618         case DEVLINK_PARAM_TYPE_STRING:
3619                 return NLA_STRING;
3620         case DEVLINK_PARAM_TYPE_BOOL:
3621                 return NLA_FLAG;
3622         default:
3623                 return -EINVAL;
3624         }
3625 }
3626
3627 static int
3628 devlink_nl_param_value_fill_one(struct sk_buff *msg,
3629                                 enum devlink_param_type type,
3630                                 enum devlink_param_cmode cmode,
3631                                 union devlink_param_value val)
3632 {
3633         struct nlattr *param_value_attr;
3634
3635         param_value_attr = nla_nest_start_noflag(msg,
3636                                                  DEVLINK_ATTR_PARAM_VALUE);
3637         if (!param_value_attr)
3638                 goto nla_put_failure;
3639
3640         if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_CMODE, cmode))
3641                 goto value_nest_cancel;
3642
3643         switch (type) {
3644         case DEVLINK_PARAM_TYPE_U8:
3645                 if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu8))
3646                         goto value_nest_cancel;
3647                 break;
3648         case DEVLINK_PARAM_TYPE_U16:
3649                 if (nla_put_u16(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu16))
3650                         goto value_nest_cancel;
3651                 break;
3652         case DEVLINK_PARAM_TYPE_U32:
3653                 if (nla_put_u32(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu32))
3654                         goto value_nest_cancel;
3655                 break;
3656         case DEVLINK_PARAM_TYPE_STRING:
3657                 if (nla_put_string(msg, DEVLINK_ATTR_PARAM_VALUE_DATA,
3658                                    val.vstr))
3659                         goto value_nest_cancel;
3660                 break;
3661         case DEVLINK_PARAM_TYPE_BOOL:
3662                 if (val.vbool &&
3663                     nla_put_flag(msg, DEVLINK_ATTR_PARAM_VALUE_DATA))
3664                         goto value_nest_cancel;
3665                 break;
3666         }
3667
3668         nla_nest_end(msg, param_value_attr);
3669         return 0;
3670
3671 value_nest_cancel:
3672         nla_nest_cancel(msg, param_value_attr);
3673 nla_put_failure:
3674         return -EMSGSIZE;
3675 }
3676
3677 static int devlink_nl_param_fill(struct sk_buff *msg, struct devlink *devlink,
3678                                  unsigned int port_index,
3679                                  struct devlink_param_item *param_item,
3680                                  enum devlink_command cmd,
3681                                  u32 portid, u32 seq, int flags)
3682 {
3683         union devlink_param_value param_value[DEVLINK_PARAM_CMODE_MAX + 1];
3684         bool param_value_set[DEVLINK_PARAM_CMODE_MAX + 1] = {};
3685         const struct devlink_param *param = param_item->param;
3686         struct devlink_param_gset_ctx ctx;
3687         struct nlattr *param_values_list;
3688         struct nlattr *param_attr;
3689         int nla_type;
3690         void *hdr;
3691         int err;
3692         int i;
3693
3694         /* Get value from driver part to driverinit configuration mode */
3695         for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
3696                 if (!devlink_param_cmode_is_supported(param, i))
3697                         continue;
3698                 if (i == DEVLINK_PARAM_CMODE_DRIVERINIT) {
3699                         if (!param_item->driverinit_value_valid)
3700                                 return -EOPNOTSUPP;
3701                         param_value[i] = param_item->driverinit_value;
3702                 } else {
3703                         if (!param_item->published)
3704                                 continue;
3705                         ctx.cmode = i;
3706                         err = devlink_param_get(devlink, param, &ctx);
3707                         if (err)
3708                                 return err;
3709                         param_value[i] = ctx.val;
3710                 }
3711                 param_value_set[i] = true;
3712         }
3713
3714         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
3715         if (!hdr)
3716                 return -EMSGSIZE;
3717
3718         if (devlink_nl_put_handle(msg, devlink))
3719                 goto genlmsg_cancel;
3720
3721         if (cmd == DEVLINK_CMD_PORT_PARAM_GET ||
3722             cmd == DEVLINK_CMD_PORT_PARAM_NEW ||
3723             cmd == DEVLINK_CMD_PORT_PARAM_DEL)
3724                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, port_index))
3725                         goto genlmsg_cancel;
3726
3727         param_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PARAM);
3728         if (!param_attr)
3729                 goto genlmsg_cancel;
3730         if (nla_put_string(msg, DEVLINK_ATTR_PARAM_NAME, param->name))
3731                 goto param_nest_cancel;
3732         if (param->generic && nla_put_flag(msg, DEVLINK_ATTR_PARAM_GENERIC))
3733                 goto param_nest_cancel;
3734
3735         nla_type = devlink_param_type_to_nla_type(param->type);
3736         if (nla_type < 0)
3737                 goto param_nest_cancel;
3738         if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_TYPE, nla_type))
3739                 goto param_nest_cancel;
3740
3741         param_values_list = nla_nest_start_noflag(msg,
3742                                                   DEVLINK_ATTR_PARAM_VALUES_LIST);
3743         if (!param_values_list)
3744                 goto param_nest_cancel;
3745
3746         for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
3747                 if (!param_value_set[i])
3748                         continue;
3749                 err = devlink_nl_param_value_fill_one(msg, param->type,
3750                                                       i, param_value[i]);
3751                 if (err)
3752                         goto values_list_nest_cancel;
3753         }
3754
3755         nla_nest_end(msg, param_values_list);
3756         nla_nest_end(msg, param_attr);
3757         genlmsg_end(msg, hdr);
3758         return 0;
3759
3760 values_list_nest_cancel:
3761         nla_nest_end(msg, param_values_list);
3762 param_nest_cancel:
3763         nla_nest_cancel(msg, param_attr);
3764 genlmsg_cancel:
3765         genlmsg_cancel(msg, hdr);
3766         return -EMSGSIZE;
3767 }
3768
3769 static void devlink_param_notify(struct devlink *devlink,
3770                                  unsigned int port_index,
3771                                  struct devlink_param_item *param_item,
3772                                  enum devlink_command cmd)
3773 {
3774         struct sk_buff *msg;
3775         int err;
3776
3777         WARN_ON(cmd != DEVLINK_CMD_PARAM_NEW && cmd != DEVLINK_CMD_PARAM_DEL &&
3778                 cmd != DEVLINK_CMD_PORT_PARAM_NEW &&
3779                 cmd != DEVLINK_CMD_PORT_PARAM_DEL);
3780
3781         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
3782         if (!msg)
3783                 return;
3784         err = devlink_nl_param_fill(msg, devlink, port_index, param_item, cmd,
3785                                     0, 0, 0);
3786         if (err) {
3787                 nlmsg_free(msg);
3788                 return;
3789         }
3790
3791         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
3792                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
3793 }
3794
3795 static int devlink_nl_cmd_param_get_dumpit(struct sk_buff *msg,
3796                                            struct netlink_callback *cb)
3797 {
3798         struct devlink_param_item *param_item;
3799         struct devlink *devlink;
3800         int start = cb->args[0];
3801         int idx = 0;
3802         int err = 0;
3803
3804         mutex_lock(&devlink_mutex);
3805         list_for_each_entry(devlink, &devlink_list, list) {
3806                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
3807                         continue;
3808                 mutex_lock(&devlink->lock);
3809                 list_for_each_entry(param_item, &devlink->param_list, list) {
3810                         if (idx < start) {
3811                                 idx++;
3812                                 continue;
3813                         }
3814                         err = devlink_nl_param_fill(msg, devlink, 0, param_item,
3815                                                     DEVLINK_CMD_PARAM_GET,
3816                                                     NETLINK_CB(cb->skb).portid,
3817                                                     cb->nlh->nlmsg_seq,
3818                                                     NLM_F_MULTI);
3819                         if (err == -EOPNOTSUPP) {
3820                                 err = 0;
3821                         } else if (err) {
3822                                 mutex_unlock(&devlink->lock);
3823                                 goto out;
3824                         }
3825                         idx++;
3826                 }
3827                 mutex_unlock(&devlink->lock);
3828         }
3829 out:
3830         mutex_unlock(&devlink_mutex);
3831
3832         if (err != -EMSGSIZE)
3833                 return err;
3834
3835         cb->args[0] = idx;
3836         return msg->len;
3837 }
3838
3839 static int
3840 devlink_param_type_get_from_info(struct genl_info *info,
3841                                  enum devlink_param_type *param_type)
3842 {
3843         if (!info->attrs[DEVLINK_ATTR_PARAM_TYPE])
3844                 return -EINVAL;
3845
3846         switch (nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_TYPE])) {
3847         case NLA_U8:
3848                 *param_type = DEVLINK_PARAM_TYPE_U8;
3849                 break;
3850         case NLA_U16:
3851                 *param_type = DEVLINK_PARAM_TYPE_U16;
3852                 break;
3853         case NLA_U32:
3854                 *param_type = DEVLINK_PARAM_TYPE_U32;
3855                 break;
3856         case NLA_STRING:
3857                 *param_type = DEVLINK_PARAM_TYPE_STRING;
3858                 break;
3859         case NLA_FLAG:
3860                 *param_type = DEVLINK_PARAM_TYPE_BOOL;
3861                 break;
3862         default:
3863                 return -EINVAL;
3864         }
3865
3866         return 0;
3867 }
3868
3869 static int
3870 devlink_param_value_get_from_info(const struct devlink_param *param,
3871                                   struct genl_info *info,
3872                                   union devlink_param_value *value)
3873 {
3874         struct nlattr *param_data;
3875         int len;
3876
3877         param_data = info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA];
3878
3879         if (param->type != DEVLINK_PARAM_TYPE_BOOL && !param_data)
3880                 return -EINVAL;
3881
3882         switch (param->type) {
3883         case DEVLINK_PARAM_TYPE_U8:
3884                 if (nla_len(param_data) != sizeof(u8))
3885                         return -EINVAL;
3886                 value->vu8 = nla_get_u8(param_data);
3887                 break;
3888         case DEVLINK_PARAM_TYPE_U16:
3889                 if (nla_len(param_data) != sizeof(u16))
3890                         return -EINVAL;
3891                 value->vu16 = nla_get_u16(param_data);
3892                 break;
3893         case DEVLINK_PARAM_TYPE_U32:
3894                 if (nla_len(param_data) != sizeof(u32))
3895                         return -EINVAL;
3896                 value->vu32 = nla_get_u32(param_data);
3897                 break;
3898         case DEVLINK_PARAM_TYPE_STRING:
3899                 len = strnlen(nla_data(param_data), nla_len(param_data));
3900                 if (len == nla_len(param_data) ||
3901                     len >= __DEVLINK_PARAM_MAX_STRING_VALUE)
3902                         return -EINVAL;
3903                 strcpy(value->vstr, nla_data(param_data));
3904                 break;
3905         case DEVLINK_PARAM_TYPE_BOOL:
3906                 if (param_data && nla_len(param_data))
3907                         return -EINVAL;
3908                 value->vbool = nla_get_flag(param_data);
3909                 break;
3910         }
3911         return 0;
3912 }
3913
3914 static struct devlink_param_item *
3915 devlink_param_get_from_info(struct list_head *param_list,
3916                             struct genl_info *info)
3917 {
3918         char *param_name;
3919
3920         if (!info->attrs[DEVLINK_ATTR_PARAM_NAME])
3921                 return NULL;
3922
3923         param_name = nla_data(info->attrs[DEVLINK_ATTR_PARAM_NAME]);
3924         return devlink_param_find_by_name(param_list, param_name);
3925 }
3926
3927 static int devlink_nl_cmd_param_get_doit(struct sk_buff *skb,
3928                                          struct genl_info *info)
3929 {
3930         struct devlink *devlink = info->user_ptr[0];
3931         struct devlink_param_item *param_item;
3932         struct sk_buff *msg;
3933         int err;
3934
3935         param_item = devlink_param_get_from_info(&devlink->param_list, info);
3936         if (!param_item)
3937                 return -EINVAL;
3938
3939         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
3940         if (!msg)
3941                 return -ENOMEM;
3942
3943         err = devlink_nl_param_fill(msg, devlink, 0, param_item,
3944                                     DEVLINK_CMD_PARAM_GET,
3945                                     info->snd_portid, info->snd_seq, 0);
3946         if (err) {
3947                 nlmsg_free(msg);
3948                 return err;
3949         }
3950
3951         return genlmsg_reply(msg, info);
3952 }
3953
3954 static int __devlink_nl_cmd_param_set_doit(struct devlink *devlink,
3955                                            unsigned int port_index,
3956                                            struct list_head *param_list,
3957                                            struct genl_info *info,
3958                                            enum devlink_command cmd)
3959 {
3960         enum devlink_param_type param_type;
3961         struct devlink_param_gset_ctx ctx;
3962         enum devlink_param_cmode cmode;
3963         struct devlink_param_item *param_item;
3964         const struct devlink_param *param;
3965         union devlink_param_value value;
3966         int err = 0;
3967
3968         param_item = devlink_param_get_from_info(param_list, info);
3969         if (!param_item)
3970                 return -EINVAL;
3971         param = param_item->param;
3972         err = devlink_param_type_get_from_info(info, &param_type);
3973         if (err)
3974                 return err;
3975         if (param_type != param->type)
3976                 return -EINVAL;
3977         err = devlink_param_value_get_from_info(param, info, &value);
3978         if (err)
3979                 return err;
3980         if (param->validate) {
3981                 err = param->validate(devlink, param->id, value, info->extack);
3982                 if (err)
3983                         return err;
3984         }
3985
3986         if (!info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE])
3987                 return -EINVAL;
3988         cmode = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]);
3989         if (!devlink_param_cmode_is_supported(param, cmode))
3990                 return -EOPNOTSUPP;
3991
3992         if (cmode == DEVLINK_PARAM_CMODE_DRIVERINIT) {
3993                 if (param->type == DEVLINK_PARAM_TYPE_STRING)
3994                         strcpy(param_item->driverinit_value.vstr, value.vstr);
3995                 else
3996                         param_item->driverinit_value = value;
3997                 param_item->driverinit_value_valid = true;
3998         } else {
3999                 if (!param->set)
4000                         return -EOPNOTSUPP;
4001                 ctx.val = value;
4002                 ctx.cmode = cmode;
4003                 err = devlink_param_set(devlink, param, &ctx);
4004                 if (err)
4005                         return err;
4006         }
4007
4008         devlink_param_notify(devlink, port_index, param_item, cmd);
4009         return 0;
4010 }
4011
4012 static int devlink_nl_cmd_param_set_doit(struct sk_buff *skb,
4013                                          struct genl_info *info)
4014 {
4015         struct devlink *devlink = info->user_ptr[0];
4016
4017         return __devlink_nl_cmd_param_set_doit(devlink, 0, &devlink->param_list,
4018                                                info, DEVLINK_CMD_PARAM_NEW);
4019 }
4020
4021 static int devlink_param_register_one(struct devlink *devlink,
4022                                       unsigned int port_index,
4023                                       struct list_head *param_list,
4024                                       const struct devlink_param *param,
4025                                       enum devlink_command cmd)
4026 {
4027         struct devlink_param_item *param_item;
4028
4029         if (devlink_param_find_by_name(param_list, param->name))
4030                 return -EEXIST;
4031
4032         if (param->supported_cmodes == BIT(DEVLINK_PARAM_CMODE_DRIVERINIT))
4033                 WARN_ON(param->get || param->set);
4034         else
4035                 WARN_ON(!param->get || !param->set);
4036
4037         param_item = kzalloc(sizeof(*param_item), GFP_KERNEL);
4038         if (!param_item)
4039                 return -ENOMEM;
4040         param_item->param = param;
4041
4042         list_add_tail(&param_item->list, param_list);
4043         devlink_param_notify(devlink, port_index, param_item, cmd);
4044         return 0;
4045 }
4046
4047 static void devlink_param_unregister_one(struct devlink *devlink,
4048                                          unsigned int port_index,
4049                                          struct list_head *param_list,
4050                                          const struct devlink_param *param,
4051                                          enum devlink_command cmd)
4052 {
4053         struct devlink_param_item *param_item;
4054
4055         param_item = devlink_param_find_by_name(param_list, param->name);
4056         WARN_ON(!param_item);
4057         devlink_param_notify(devlink, port_index, param_item, cmd);
4058         list_del(&param_item->list);
4059         kfree(param_item);
4060 }
4061
4062 static int devlink_nl_cmd_port_param_get_dumpit(struct sk_buff *msg,
4063                                                 struct netlink_callback *cb)
4064 {
4065         struct devlink_param_item *param_item;
4066         struct devlink_port *devlink_port;
4067         struct devlink *devlink;
4068         int start = cb->args[0];
4069         int idx = 0;
4070         int err = 0;
4071
4072         mutex_lock(&devlink_mutex);
4073         list_for_each_entry(devlink, &devlink_list, list) {
4074                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
4075                         continue;
4076                 mutex_lock(&devlink->lock);
4077                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
4078                         list_for_each_entry(param_item,
4079                                             &devlink_port->param_list, list) {
4080                                 if (idx < start) {
4081                                         idx++;
4082                                         continue;
4083                                 }
4084                                 err = devlink_nl_param_fill(msg,
4085                                                 devlink_port->devlink,
4086                                                 devlink_port->index, param_item,
4087                                                 DEVLINK_CMD_PORT_PARAM_GET,
4088                                                 NETLINK_CB(cb->skb).portid,
4089                                                 cb->nlh->nlmsg_seq,
4090                                                 NLM_F_MULTI);
4091                                 if (err == -EOPNOTSUPP) {
4092                                         err = 0;
4093                                 } else if (err) {
4094                                         mutex_unlock(&devlink->lock);
4095                                         goto out;
4096                                 }
4097                                 idx++;
4098                         }
4099                 }
4100                 mutex_unlock(&devlink->lock);
4101         }
4102 out:
4103         mutex_unlock(&devlink_mutex);
4104
4105         if (err != -EMSGSIZE)
4106                 return err;
4107
4108         cb->args[0] = idx;
4109         return msg->len;
4110 }
4111
4112 static int devlink_nl_cmd_port_param_get_doit(struct sk_buff *skb,
4113                                               struct genl_info *info)
4114 {
4115         struct devlink_port *devlink_port = info->user_ptr[0];
4116         struct devlink_param_item *param_item;
4117         struct sk_buff *msg;
4118         int err;
4119
4120         param_item = devlink_param_get_from_info(&devlink_port->param_list,
4121                                                  info);
4122         if (!param_item)
4123                 return -EINVAL;
4124
4125         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4126         if (!msg)
4127                 return -ENOMEM;
4128
4129         err = devlink_nl_param_fill(msg, devlink_port->devlink,
4130                                     devlink_port->index, param_item,
4131                                     DEVLINK_CMD_PORT_PARAM_GET,
4132                                     info->snd_portid, info->snd_seq, 0);
4133         if (err) {
4134                 nlmsg_free(msg);
4135                 return err;
4136         }
4137
4138         return genlmsg_reply(msg, info);
4139 }
4140
4141 static int devlink_nl_cmd_port_param_set_doit(struct sk_buff *skb,
4142                                               struct genl_info *info)
4143 {
4144         struct devlink_port *devlink_port = info->user_ptr[0];
4145
4146         return __devlink_nl_cmd_param_set_doit(devlink_port->devlink,
4147                                                devlink_port->index,
4148                                                &devlink_port->param_list, info,
4149                                                DEVLINK_CMD_PORT_PARAM_NEW);
4150 }
4151
4152 static int devlink_nl_region_snapshot_id_put(struct sk_buff *msg,
4153                                              struct devlink *devlink,
4154                                              struct devlink_snapshot *snapshot)
4155 {
4156         struct nlattr *snap_attr;
4157         int err;
4158
4159         snap_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_SNAPSHOT);
4160         if (!snap_attr)
4161                 return -EINVAL;
4162
4163         err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID, snapshot->id);
4164         if (err)
4165                 goto nla_put_failure;
4166
4167         nla_nest_end(msg, snap_attr);
4168         return 0;
4169
4170 nla_put_failure:
4171         nla_nest_cancel(msg, snap_attr);
4172         return err;
4173 }
4174
4175 static int devlink_nl_region_snapshots_id_put(struct sk_buff *msg,
4176                                               struct devlink *devlink,
4177                                               struct devlink_region *region)
4178 {
4179         struct devlink_snapshot *snapshot;
4180         struct nlattr *snapshots_attr;
4181         int err;
4182
4183         snapshots_attr = nla_nest_start_noflag(msg,
4184                                                DEVLINK_ATTR_REGION_SNAPSHOTS);
4185         if (!snapshots_attr)
4186                 return -EINVAL;
4187
4188         list_for_each_entry(snapshot, &region->snapshot_list, list) {
4189                 err = devlink_nl_region_snapshot_id_put(msg, devlink, snapshot);
4190                 if (err)
4191                         goto nla_put_failure;
4192         }
4193
4194         nla_nest_end(msg, snapshots_attr);
4195         return 0;
4196
4197 nla_put_failure:
4198         nla_nest_cancel(msg, snapshots_attr);
4199         return err;
4200 }
4201
4202 static int devlink_nl_region_fill(struct sk_buff *msg, struct devlink *devlink,
4203                                   enum devlink_command cmd, u32 portid,
4204                                   u32 seq, int flags,
4205                                   struct devlink_region *region)
4206 {
4207         void *hdr;
4208         int err;
4209
4210         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
4211         if (!hdr)
4212                 return -EMSGSIZE;
4213
4214         err = devlink_nl_put_handle(msg, devlink);
4215         if (err)
4216                 goto nla_put_failure;
4217
4218         if (region->port) {
4219                 err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
4220                                   region->port->index);
4221                 if (err)
4222                         goto nla_put_failure;
4223         }
4224
4225         err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME, region->ops->name);
4226         if (err)
4227                 goto nla_put_failure;
4228
4229         err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
4230                                 region->size,
4231                                 DEVLINK_ATTR_PAD);
4232         if (err)
4233                 goto nla_put_failure;
4234
4235         err = devlink_nl_region_snapshots_id_put(msg, devlink, region);
4236         if (err)
4237                 goto nla_put_failure;
4238
4239         genlmsg_end(msg, hdr);
4240         return 0;
4241
4242 nla_put_failure:
4243         genlmsg_cancel(msg, hdr);
4244         return err;
4245 }
4246
4247 static struct sk_buff *
4248 devlink_nl_region_notify_build(struct devlink_region *region,
4249                                struct devlink_snapshot *snapshot,
4250                                enum devlink_command cmd, u32 portid, u32 seq)
4251 {
4252         struct devlink *devlink = region->devlink;
4253         struct sk_buff *msg;
4254         void *hdr;
4255         int err;
4256
4257
4258         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4259         if (!msg)
4260                 return ERR_PTR(-ENOMEM);
4261
4262         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, 0, cmd);
4263         if (!hdr) {
4264                 err = -EMSGSIZE;
4265                 goto out_free_msg;
4266         }
4267
4268         err = devlink_nl_put_handle(msg, devlink);
4269         if (err)
4270                 goto out_cancel_msg;
4271
4272         if (region->port) {
4273                 err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
4274                                   region->port->index);
4275                 if (err)
4276                         goto out_cancel_msg;
4277         }
4278
4279         err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME,
4280                              region->ops->name);
4281         if (err)
4282                 goto out_cancel_msg;
4283
4284         if (snapshot) {
4285                 err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID,
4286                                   snapshot->id);
4287                 if (err)
4288                         goto out_cancel_msg;
4289         } else {
4290                 err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
4291                                         region->size, DEVLINK_ATTR_PAD);
4292                 if (err)
4293                         goto out_cancel_msg;
4294         }
4295         genlmsg_end(msg, hdr);
4296
4297         return msg;
4298
4299 out_cancel_msg:
4300         genlmsg_cancel(msg, hdr);
4301 out_free_msg:
4302         nlmsg_free(msg);
4303         return ERR_PTR(err);
4304 }
4305
4306 static void devlink_nl_region_notify(struct devlink_region *region,
4307                                      struct devlink_snapshot *snapshot,
4308                                      enum devlink_command cmd)
4309 {
4310         struct devlink *devlink = region->devlink;
4311         struct sk_buff *msg;
4312
4313         WARN_ON(cmd != DEVLINK_CMD_REGION_NEW && cmd != DEVLINK_CMD_REGION_DEL);
4314
4315         msg = devlink_nl_region_notify_build(region, snapshot, cmd, 0, 0);
4316         if (IS_ERR(msg))
4317                 return;
4318
4319         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
4320                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
4321 }
4322
4323 /**
4324  * __devlink_snapshot_id_increment - Increment number of snapshots using an id
4325  *      @devlink: devlink instance
4326  *      @id: the snapshot id
4327  *
4328  *      Track when a new snapshot begins using an id. Load the count for the
4329  *      given id from the snapshot xarray, increment it, and store it back.
4330  *
4331  *      Called when a new snapshot is created with the given id.
4332  *
4333  *      The id *must* have been previously allocated by
4334  *      devlink_region_snapshot_id_get().
4335  *
4336  *      Returns 0 on success, or an error on failure.
4337  */
4338 static int __devlink_snapshot_id_increment(struct devlink *devlink, u32 id)
4339 {
4340         unsigned long count;
4341         void *p;
4342
4343         lockdep_assert_held(&devlink->lock);
4344
4345         p = xa_load(&devlink->snapshot_ids, id);
4346         if (WARN_ON(!p))
4347                 return -EINVAL;
4348
4349         if (WARN_ON(!xa_is_value(p)))
4350                 return -EINVAL;
4351
4352         count = xa_to_value(p);
4353         count++;
4354
4355         return xa_err(xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
4356                                GFP_KERNEL));
4357 }
4358
4359 /**
4360  * __devlink_snapshot_id_decrement - Decrease number of snapshots using an id
4361  *      @devlink: devlink instance
4362  *      @id: the snapshot id
4363  *
4364  *      Track when a snapshot is deleted and stops using an id. Load the count
4365  *      for the given id from the snapshot xarray, decrement it, and store it
4366  *      back.
4367  *
4368  *      If the count reaches zero, erase this id from the xarray, freeing it
4369  *      up for future re-use by devlink_region_snapshot_id_get().
4370  *
4371  *      Called when a snapshot using the given id is deleted, and when the
4372  *      initial allocator of the id is finished using it.
4373  */
4374 static void __devlink_snapshot_id_decrement(struct devlink *devlink, u32 id)
4375 {
4376         unsigned long count;
4377         void *p;
4378
4379         lockdep_assert_held(&devlink->lock);
4380
4381         p = xa_load(&devlink->snapshot_ids, id);
4382         if (WARN_ON(!p))
4383                 return;
4384
4385         if (WARN_ON(!xa_is_value(p)))
4386                 return;
4387
4388         count = xa_to_value(p);
4389
4390         if (count > 1) {
4391                 count--;
4392                 xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
4393                          GFP_KERNEL);
4394         } else {
4395                 /* If this was the last user, we can erase this id */
4396                 xa_erase(&devlink->snapshot_ids, id);
4397         }
4398 }
4399
4400 /**
4401  *      __devlink_snapshot_id_insert - Insert a specific snapshot ID
4402  *      @devlink: devlink instance
4403  *      @id: the snapshot id
4404  *
4405  *      Mark the given snapshot id as used by inserting a zero value into the
4406  *      snapshot xarray.
4407  *
4408  *      This must be called while holding the devlink instance lock. Unlike
4409  *      devlink_snapshot_id_get, the initial reference count is zero, not one.
4410  *      It is expected that the id will immediately be used before
4411  *      releasing the devlink instance lock.
4412  *
4413  *      Returns zero on success, or an error code if the snapshot id could not
4414  *      be inserted.
4415  */
4416 static int __devlink_snapshot_id_insert(struct devlink *devlink, u32 id)
4417 {
4418         lockdep_assert_held(&devlink->lock);
4419
4420         if (xa_load(&devlink->snapshot_ids, id))
4421                 return -EEXIST;
4422
4423         return xa_err(xa_store(&devlink->snapshot_ids, id, xa_mk_value(0),
4424                                GFP_KERNEL));
4425 }
4426
4427 /**
4428  *      __devlink_region_snapshot_id_get - get snapshot ID
4429  *      @devlink: devlink instance
4430  *      @id: storage to return snapshot id
4431  *
4432  *      Allocates a new snapshot id. Returns zero on success, or a negative
4433  *      error on failure. Must be called while holding the devlink instance
4434  *      lock.
4435  *
4436  *      Snapshot IDs are tracked using an xarray which stores the number of
4437  *      users of the snapshot id.
4438  *
4439  *      Note that the caller of this function counts as a 'user', in order to
4440  *      avoid race conditions. The caller must release its hold on the
4441  *      snapshot by using devlink_region_snapshot_id_put.
4442  */
4443 static int __devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
4444 {
4445         lockdep_assert_held(&devlink->lock);
4446
4447         return xa_alloc(&devlink->snapshot_ids, id, xa_mk_value(1),
4448                         xa_limit_32b, GFP_KERNEL);
4449 }
4450
4451 /**
4452  *      __devlink_region_snapshot_create - create a new snapshot
4453  *      This will add a new snapshot of a region. The snapshot
4454  *      will be stored on the region struct and can be accessed
4455  *      from devlink. This is useful for future analyses of snapshots.
4456  *      Multiple snapshots can be created on a region.
4457  *      The @snapshot_id should be obtained using the getter function.
4458  *
4459  *      Must be called only while holding the devlink instance lock.
4460  *
4461  *      @region: devlink region of the snapshot
4462  *      @data: snapshot data
4463  *      @snapshot_id: snapshot id to be created
4464  */
4465 static int
4466 __devlink_region_snapshot_create(struct devlink_region *region,
4467                                  u8 *data, u32 snapshot_id)
4468 {
4469         struct devlink *devlink = region->devlink;
4470         struct devlink_snapshot *snapshot;
4471         int err;
4472
4473         lockdep_assert_held(&devlink->lock);
4474
4475         /* check if region can hold one more snapshot */
4476         if (region->cur_snapshots == region->max_snapshots)
4477                 return -ENOSPC;
4478
4479         if (devlink_region_snapshot_get_by_id(region, snapshot_id))
4480                 return -EEXIST;
4481
4482         snapshot = kzalloc(sizeof(*snapshot), GFP_KERNEL);
4483         if (!snapshot)
4484                 return -ENOMEM;
4485
4486         err = __devlink_snapshot_id_increment(devlink, snapshot_id);
4487         if (err)
4488                 goto err_snapshot_id_increment;
4489
4490         snapshot->id = snapshot_id;
4491         snapshot->region = region;
4492         snapshot->data = data;
4493
4494         list_add_tail(&snapshot->list, &region->snapshot_list);
4495
4496         region->cur_snapshots++;
4497
4498         devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_NEW);
4499         return 0;
4500
4501 err_snapshot_id_increment:
4502         kfree(snapshot);
4503         return err;
4504 }
4505
4506 static void devlink_region_snapshot_del(struct devlink_region *region,
4507                                         struct devlink_snapshot *snapshot)
4508 {
4509         struct devlink *devlink = region->devlink;
4510
4511         lockdep_assert_held(&devlink->lock);
4512
4513         devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_DEL);
4514         region->cur_snapshots--;
4515         list_del(&snapshot->list);
4516         region->ops->destructor(snapshot->data);
4517         __devlink_snapshot_id_decrement(devlink, snapshot->id);
4518         kfree(snapshot);
4519 }
4520
4521 static int devlink_nl_cmd_region_get_doit(struct sk_buff *skb,
4522                                           struct genl_info *info)
4523 {
4524         struct devlink *devlink = info->user_ptr[0];
4525         struct devlink_port *port = NULL;
4526         struct devlink_region *region;
4527         const char *region_name;
4528         struct sk_buff *msg;
4529         unsigned int index;
4530         int err;
4531
4532         if (!info->attrs[DEVLINK_ATTR_REGION_NAME])
4533                 return -EINVAL;
4534
4535         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
4536                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
4537
4538                 port = devlink_port_get_by_index(devlink, index);
4539                 if (!port)
4540                         return -ENODEV;
4541         }
4542
4543         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
4544         if (port)
4545                 region = devlink_port_region_get_by_name(port, region_name);
4546         else
4547                 region = devlink_region_get_by_name(devlink, region_name);
4548
4549         if (!region)
4550                 return -EINVAL;
4551
4552         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4553         if (!msg)
4554                 return -ENOMEM;
4555
4556         err = devlink_nl_region_fill(msg, devlink, DEVLINK_CMD_REGION_GET,
4557                                      info->snd_portid, info->snd_seq, 0,
4558                                      region);
4559         if (err) {
4560                 nlmsg_free(msg);
4561                 return err;
4562         }
4563
4564         return genlmsg_reply(msg, info);
4565 }
4566
4567 static int devlink_nl_cmd_region_get_port_dumpit(struct sk_buff *msg,
4568                                                  struct netlink_callback *cb,
4569                                                  struct devlink_port *port,
4570                                                  int *idx,
4571                                                  int start)
4572 {
4573         struct devlink_region *region;
4574         int err = 0;
4575
4576         list_for_each_entry(region, &port->region_list, list) {
4577                 if (*idx < start) {
4578                         (*idx)++;
4579                         continue;
4580                 }
4581                 err = devlink_nl_region_fill(msg, port->devlink,
4582                                              DEVLINK_CMD_REGION_GET,
4583                                              NETLINK_CB(cb->skb).portid,
4584                                              cb->nlh->nlmsg_seq,
4585                                              NLM_F_MULTI, region);
4586                 if (err)
4587                         goto out;
4588                 (*idx)++;
4589         }
4590
4591 out:
4592         return err;
4593 }
4594
4595 static int devlink_nl_cmd_region_get_devlink_dumpit(struct sk_buff *msg,
4596                                                     struct netlink_callback *cb,
4597                                                     struct devlink *devlink,
4598                                                     int *idx,
4599                                                     int start)
4600 {
4601         struct devlink_region *region;
4602         struct devlink_port *port;
4603         int err = 0;
4604
4605         mutex_lock(&devlink->lock);
4606         list_for_each_entry(region, &devlink->region_list, list) {
4607                 if (*idx < start) {
4608                         (*idx)++;
4609                         continue;
4610                 }
4611                 err = devlink_nl_region_fill(msg, devlink,
4612                                              DEVLINK_CMD_REGION_GET,
4613                                              NETLINK_CB(cb->skb).portid,
4614                                              cb->nlh->nlmsg_seq,
4615                                              NLM_F_MULTI, region);
4616                 if (err)
4617                         goto out;
4618                 (*idx)++;
4619         }
4620
4621         list_for_each_entry(port, &devlink->port_list, list) {
4622                 err = devlink_nl_cmd_region_get_port_dumpit(msg, cb, port, idx,
4623                                                             start);
4624                 if (err)
4625                         goto out;
4626         }
4627
4628 out:
4629         mutex_unlock(&devlink->lock);
4630         return err;
4631 }
4632
4633 static int devlink_nl_cmd_region_get_dumpit(struct sk_buff *msg,
4634                                             struct netlink_callback *cb)
4635 {
4636         struct devlink *devlink;
4637         int start = cb->args[0];
4638         int idx = 0;
4639         int err;
4640
4641         mutex_lock(&devlink_mutex);
4642         list_for_each_entry(devlink, &devlink_list, list) {
4643                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
4644                         continue;
4645                 err = devlink_nl_cmd_region_get_devlink_dumpit(msg, cb, devlink,
4646                                                                &idx, start);
4647                 if (err)
4648                         goto out;
4649         }
4650 out:
4651         mutex_unlock(&devlink_mutex);
4652         cb->args[0] = idx;
4653         return msg->len;
4654 }
4655
4656 static int devlink_nl_cmd_region_del(struct sk_buff *skb,
4657                                      struct genl_info *info)
4658 {
4659         struct devlink *devlink = info->user_ptr[0];
4660         struct devlink_snapshot *snapshot;
4661         struct devlink_port *port = NULL;
4662         struct devlink_region *region;
4663         const char *region_name;
4664         unsigned int index;
4665         u32 snapshot_id;
4666
4667         if (!info->attrs[DEVLINK_ATTR_REGION_NAME] ||
4668             !info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID])
4669                 return -EINVAL;
4670
4671         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
4672         snapshot_id = nla_get_u32(info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
4673
4674         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
4675                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
4676
4677                 port = devlink_port_get_by_index(devlink, index);
4678                 if (!port)
4679                         return -ENODEV;
4680         }
4681
4682         if (port)
4683                 region = devlink_port_region_get_by_name(port, region_name);
4684         else
4685                 region = devlink_region_get_by_name(devlink, region_name);
4686
4687         if (!region)
4688                 return -EINVAL;
4689
4690         snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
4691         if (!snapshot)
4692                 return -EINVAL;
4693
4694         devlink_region_snapshot_del(region, snapshot);
4695         return 0;
4696 }
4697
4698 static int
4699 devlink_nl_cmd_region_new(struct sk_buff *skb, struct genl_info *info)
4700 {
4701         struct devlink *devlink = info->user_ptr[0];
4702         struct devlink_snapshot *snapshot;
4703         struct devlink_port *port = NULL;
4704         struct nlattr *snapshot_id_attr;
4705         struct devlink_region *region;
4706         const char *region_name;
4707         unsigned int index;
4708         u32 snapshot_id;
4709         u8 *data;
4710         int err;
4711
4712         if (!info->attrs[DEVLINK_ATTR_REGION_NAME]) {
4713                 NL_SET_ERR_MSG_MOD(info->extack, "No region name provided");
4714                 return -EINVAL;
4715         }
4716
4717         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
4718
4719         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
4720                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
4721
4722                 port = devlink_port_get_by_index(devlink, index);
4723                 if (!port)
4724                         return -ENODEV;
4725         }
4726
4727         if (port)
4728                 region = devlink_port_region_get_by_name(port, region_name);
4729         else
4730                 region = devlink_region_get_by_name(devlink, region_name);
4731
4732         if (!region) {
4733                 NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not exist");
4734                 return -EINVAL;
4735         }
4736
4737         if (!region->ops->snapshot) {
4738                 NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not support taking an immediate snapshot");
4739                 return -EOPNOTSUPP;
4740         }
4741
4742         if (region->cur_snapshots == region->max_snapshots) {
4743                 NL_SET_ERR_MSG_MOD(info->extack, "The region has reached the maximum number of stored snapshots");
4744                 return -ENOSPC;
4745         }
4746
4747         snapshot_id_attr = info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID];
4748         if (snapshot_id_attr) {
4749                 snapshot_id = nla_get_u32(snapshot_id_attr);
4750
4751                 if (devlink_region_snapshot_get_by_id(region, snapshot_id)) {
4752                         NL_SET_ERR_MSG_MOD(info->extack, "The requested snapshot id is already in use");
4753                         return -EEXIST;
4754                 }
4755
4756                 err = __devlink_snapshot_id_insert(devlink, snapshot_id);
4757                 if (err)
4758                         return err;
4759         } else {
4760                 err = __devlink_region_snapshot_id_get(devlink, &snapshot_id);
4761                 if (err) {
4762                         NL_SET_ERR_MSG_MOD(info->extack, "Failed to allocate a new snapshot id");
4763                         return err;
4764                 }
4765         }
4766
4767         if (port)
4768                 err = region->port_ops->snapshot(port, region->port_ops,
4769                                                  info->extack, &data);
4770         else
4771                 err = region->ops->snapshot(devlink, region->ops,
4772                                             info->extack, &data);
4773         if (err)
4774                 goto err_snapshot_capture;
4775
4776         err = __devlink_region_snapshot_create(region, data, snapshot_id);
4777         if (err)
4778                 goto err_snapshot_create;
4779
4780         if (!snapshot_id_attr) {
4781                 struct sk_buff *msg;
4782
4783                 snapshot = devlink_region_snapshot_get_by_id(region,
4784                                                              snapshot_id);
4785                 if (WARN_ON(!snapshot))
4786                         return -EINVAL;
4787
4788                 msg = devlink_nl_region_notify_build(region, snapshot,
4789                                                      DEVLINK_CMD_REGION_NEW,
4790                                                      info->snd_portid,
4791                                                      info->snd_seq);
4792                 err = PTR_ERR_OR_ZERO(msg);
4793                 if (err)
4794                         goto err_notify;
4795
4796                 err = genlmsg_reply(msg, info);
4797                 if (err)
4798                         goto err_notify;
4799         }
4800
4801         return 0;
4802
4803 err_snapshot_create:
4804         region->ops->destructor(data);
4805 err_snapshot_capture:
4806         __devlink_snapshot_id_decrement(devlink, snapshot_id);
4807         return err;
4808
4809 err_notify:
4810         devlink_region_snapshot_del(region, snapshot);
4811         return err;
4812 }
4813
4814 static int devlink_nl_cmd_region_read_chunk_fill(struct sk_buff *msg,
4815                                                  struct devlink *devlink,
4816                                                  u8 *chunk, u32 chunk_size,
4817                                                  u64 addr)
4818 {
4819         struct nlattr *chunk_attr;
4820         int err;
4821
4822         chunk_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_CHUNK);
4823         if (!chunk_attr)
4824                 return -EINVAL;
4825
4826         err = nla_put(msg, DEVLINK_ATTR_REGION_CHUNK_DATA, chunk_size, chunk);
4827         if (err)
4828                 goto nla_put_failure;
4829
4830         err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_CHUNK_ADDR, addr,
4831                                 DEVLINK_ATTR_PAD);
4832         if (err)
4833                 goto nla_put_failure;
4834
4835         nla_nest_end(msg, chunk_attr);
4836         return 0;
4837
4838 nla_put_failure:
4839         nla_nest_cancel(msg, chunk_attr);
4840         return err;
4841 }
4842
4843 #define DEVLINK_REGION_READ_CHUNK_SIZE 256
4844
4845 static int devlink_nl_region_read_snapshot_fill(struct sk_buff *skb,
4846                                                 struct devlink *devlink,
4847                                                 struct devlink_region *region,
4848                                                 struct nlattr **attrs,
4849                                                 u64 start_offset,
4850                                                 u64 end_offset,
4851                                                 u64 *new_offset)
4852 {
4853         struct devlink_snapshot *snapshot;
4854         u64 curr_offset = start_offset;
4855         u32 snapshot_id;
4856         int err = 0;
4857
4858         *new_offset = start_offset;
4859
4860         snapshot_id = nla_get_u32(attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
4861         snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
4862         if (!snapshot)
4863                 return -EINVAL;
4864
4865         while (curr_offset < end_offset) {
4866                 u32 data_size;
4867                 u8 *data;
4868
4869                 if (end_offset - curr_offset < DEVLINK_REGION_READ_CHUNK_SIZE)
4870                         data_size = end_offset - curr_offset;
4871                 else
4872                         data_size = DEVLINK_REGION_READ_CHUNK_SIZE;
4873
4874                 data = &snapshot->data[curr_offset];
4875                 err = devlink_nl_cmd_region_read_chunk_fill(skb, devlink,
4876                                                             data, data_size,
4877                                                             curr_offset);
4878                 if (err)
4879                         break;
4880
4881                 curr_offset += data_size;
4882         }
4883         *new_offset = curr_offset;
4884
4885         return err;
4886 }
4887
4888 static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb,
4889                                              struct netlink_callback *cb)
4890 {
4891         const struct genl_dumpit_info *info = genl_dumpit_info(cb);
4892         u64 ret_offset, start_offset, end_offset = U64_MAX;
4893         struct nlattr **attrs = info->attrs;
4894         struct devlink_port *port = NULL;
4895         struct devlink_region *region;
4896         struct nlattr *chunks_attr;
4897         const char *region_name;
4898         struct devlink *devlink;
4899         unsigned int index;
4900         void *hdr;
4901         int err;
4902
4903         start_offset = *((u64 *)&cb->args[0]);
4904
4905         mutex_lock(&devlink_mutex);
4906         devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
4907         if (IS_ERR(devlink)) {
4908                 err = PTR_ERR(devlink);
4909                 goto out_dev;
4910         }
4911
4912         mutex_lock(&devlink->lock);
4913
4914         if (!attrs[DEVLINK_ATTR_REGION_NAME] ||
4915             !attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]) {
4916                 err = -EINVAL;
4917                 goto out_unlock;
4918         }
4919
4920         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
4921                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
4922
4923                 port = devlink_port_get_by_index(devlink, index);
4924                 if (!port) {
4925                         err = -ENODEV;
4926                         goto out_unlock;
4927                 }
4928         }
4929
4930         region_name = nla_data(attrs[DEVLINK_ATTR_REGION_NAME]);
4931
4932         if (port)
4933                 region = devlink_port_region_get_by_name(port, region_name);
4934         else
4935                 region = devlink_region_get_by_name(devlink, region_name);
4936
4937         if (!region) {
4938                 err = -EINVAL;
4939                 goto out_unlock;
4940         }
4941
4942         if (attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR] &&
4943             attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]) {
4944                 if (!start_offset)
4945                         start_offset =
4946                                 nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
4947
4948                 end_offset = nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
4949                 end_offset += nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]);
4950         }
4951
4952         if (end_offset > region->size)
4953                 end_offset = region->size;
4954
4955         /* return 0 if there is no further data to read */
4956         if (start_offset == end_offset) {
4957                 err = 0;
4958                 goto out_unlock;
4959         }
4960
4961         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
4962                           &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI,
4963                           DEVLINK_CMD_REGION_READ);
4964         if (!hdr) {
4965                 err = -EMSGSIZE;
4966                 goto out_unlock;
4967         }
4968
4969         err = devlink_nl_put_handle(skb, devlink);
4970         if (err)
4971                 goto nla_put_failure;
4972
4973         if (region->port) {
4974                 err = nla_put_u32(skb, DEVLINK_ATTR_PORT_INDEX,
4975                                   region->port->index);
4976                 if (err)
4977                         goto nla_put_failure;
4978         }
4979
4980         err = nla_put_string(skb, DEVLINK_ATTR_REGION_NAME, region_name);
4981         if (err)
4982                 goto nla_put_failure;
4983
4984         chunks_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_REGION_CHUNKS);
4985         if (!chunks_attr) {
4986                 err = -EMSGSIZE;
4987                 goto nla_put_failure;
4988         }
4989
4990         err = devlink_nl_region_read_snapshot_fill(skb, devlink,
4991                                                    region, attrs,
4992                                                    start_offset,
4993                                                    end_offset, &ret_offset);
4994
4995         if (err && err != -EMSGSIZE)
4996                 goto nla_put_failure;
4997
4998         /* Check if there was any progress done to prevent infinite loop */
4999         if (ret_offset == start_offset) {
5000                 err = -EINVAL;
5001                 goto nla_put_failure;
5002         }
5003
5004         *((u64 *)&cb->args[0]) = ret_offset;
5005
5006         nla_nest_end(skb, chunks_attr);
5007         genlmsg_end(skb, hdr);
5008         mutex_unlock(&devlink->lock);
5009         mutex_unlock(&devlink_mutex);
5010
5011         return skb->len;
5012
5013 nla_put_failure:
5014         genlmsg_cancel(skb, hdr);
5015 out_unlock:
5016         mutex_unlock(&devlink->lock);
5017 out_dev:
5018         mutex_unlock(&devlink_mutex);
5019         return err;
5020 }
5021
5022 struct devlink_info_req {
5023         struct sk_buff *msg;
5024 };
5025
5026 int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name)
5027 {
5028         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, name);
5029 }
5030 EXPORT_SYMBOL_GPL(devlink_info_driver_name_put);
5031
5032 int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn)
5033 {
5034         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_SERIAL_NUMBER, sn);
5035 }
5036 EXPORT_SYMBOL_GPL(devlink_info_serial_number_put);
5037
5038 int devlink_info_board_serial_number_put(struct devlink_info_req *req,
5039                                          const char *bsn)
5040 {
5041         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER,
5042                               bsn);
5043 }
5044 EXPORT_SYMBOL_GPL(devlink_info_board_serial_number_put);
5045
5046 static int devlink_info_version_put(struct devlink_info_req *req, int attr,
5047                                     const char *version_name,
5048                                     const char *version_value)
5049 {
5050         struct nlattr *nest;
5051         int err;
5052
5053         nest = nla_nest_start_noflag(req->msg, attr);
5054         if (!nest)
5055                 return -EMSGSIZE;
5056
5057         err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_NAME,
5058                              version_name);
5059         if (err)
5060                 goto nla_put_failure;
5061
5062         err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_VALUE,
5063                              version_value);
5064         if (err)
5065                 goto nla_put_failure;
5066
5067         nla_nest_end(req->msg, nest);
5068
5069         return 0;
5070
5071 nla_put_failure:
5072         nla_nest_cancel(req->msg, nest);
5073         return err;
5074 }
5075
5076 int devlink_info_version_fixed_put(struct devlink_info_req *req,
5077                                    const char *version_name,
5078                                    const char *version_value)
5079 {
5080         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_FIXED,
5081                                         version_name, version_value);
5082 }
5083 EXPORT_SYMBOL_GPL(devlink_info_version_fixed_put);
5084
5085 int devlink_info_version_stored_put(struct devlink_info_req *req,
5086                                     const char *version_name,
5087                                     const char *version_value)
5088 {
5089         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
5090                                         version_name, version_value);
5091 }
5092 EXPORT_SYMBOL_GPL(devlink_info_version_stored_put);
5093
5094 int devlink_info_version_running_put(struct devlink_info_req *req,
5095                                      const char *version_name,
5096                                      const char *version_value)
5097 {
5098         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
5099                                         version_name, version_value);
5100 }
5101 EXPORT_SYMBOL_GPL(devlink_info_version_running_put);
5102
5103 static int
5104 devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink,
5105                      enum devlink_command cmd, u32 portid,
5106                      u32 seq, int flags, struct netlink_ext_ack *extack)
5107 {
5108         struct devlink_info_req req;
5109         void *hdr;
5110         int err;
5111
5112         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
5113         if (!hdr)
5114                 return -EMSGSIZE;
5115
5116         err = -EMSGSIZE;
5117         if (devlink_nl_put_handle(msg, devlink))
5118                 goto err_cancel_msg;
5119
5120         req.msg = msg;
5121         err = devlink->ops->info_get(devlink, &req, extack);
5122         if (err)
5123                 goto err_cancel_msg;
5124
5125         genlmsg_end(msg, hdr);
5126         return 0;
5127
5128 err_cancel_msg:
5129         genlmsg_cancel(msg, hdr);
5130         return err;
5131 }
5132
5133 static int devlink_nl_cmd_info_get_doit(struct sk_buff *skb,
5134                                         struct genl_info *info)
5135 {
5136         struct devlink *devlink = info->user_ptr[0];
5137         struct sk_buff *msg;
5138         int err;
5139
5140         if (!devlink->ops->info_get)
5141                 return -EOPNOTSUPP;
5142
5143         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5144         if (!msg)
5145                 return -ENOMEM;
5146
5147         err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
5148                                    info->snd_portid, info->snd_seq, 0,
5149                                    info->extack);
5150         if (err) {
5151                 nlmsg_free(msg);
5152                 return err;
5153         }
5154
5155         return genlmsg_reply(msg, info);
5156 }
5157
5158 static int devlink_nl_cmd_info_get_dumpit(struct sk_buff *msg,
5159                                           struct netlink_callback *cb)
5160 {
5161         struct devlink *devlink;
5162         int start = cb->args[0];
5163         int idx = 0;
5164         int err = 0;
5165
5166         mutex_lock(&devlink_mutex);
5167         list_for_each_entry(devlink, &devlink_list, list) {
5168                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5169                         continue;
5170                 if (idx < start) {
5171                         idx++;
5172                         continue;
5173                 }
5174
5175                 if (!devlink->ops->info_get) {
5176                         idx++;
5177                         continue;
5178                 }
5179
5180                 mutex_lock(&devlink->lock);
5181                 err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
5182                                            NETLINK_CB(cb->skb).portid,
5183                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
5184                                            cb->extack);
5185                 mutex_unlock(&devlink->lock);
5186                 if (err == -EOPNOTSUPP)
5187                         err = 0;
5188                 else if (err)
5189                         break;
5190                 idx++;
5191         }
5192         mutex_unlock(&devlink_mutex);
5193
5194         if (err != -EMSGSIZE)
5195                 return err;
5196
5197         cb->args[0] = idx;
5198         return msg->len;
5199 }
5200
5201 struct devlink_fmsg_item {
5202         struct list_head list;
5203         int attrtype;
5204         u8 nla_type;
5205         u16 len;
5206         int value[];
5207 };
5208
5209 struct devlink_fmsg {
5210         struct list_head item_list;
5211         bool putting_binary; /* This flag forces enclosing of binary data
5212                               * in an array brackets. It forces using
5213                               * of designated API:
5214                               * devlink_fmsg_binary_pair_nest_start()
5215                               * devlink_fmsg_binary_pair_nest_end()
5216                               */
5217 };
5218
5219 static struct devlink_fmsg *devlink_fmsg_alloc(void)
5220 {
5221         struct devlink_fmsg *fmsg;
5222
5223         fmsg = kzalloc(sizeof(*fmsg), GFP_KERNEL);
5224         if (!fmsg)
5225                 return NULL;
5226
5227         INIT_LIST_HEAD(&fmsg->item_list);
5228
5229         return fmsg;
5230 }
5231
5232 static void devlink_fmsg_free(struct devlink_fmsg *fmsg)
5233 {
5234         struct devlink_fmsg_item *item, *tmp;
5235
5236         list_for_each_entry_safe(item, tmp, &fmsg->item_list, list) {
5237                 list_del(&item->list);
5238                 kfree(item);
5239         }
5240         kfree(fmsg);
5241 }
5242
5243 static int devlink_fmsg_nest_common(struct devlink_fmsg *fmsg,
5244                                     int attrtype)
5245 {
5246         struct devlink_fmsg_item *item;
5247
5248         item = kzalloc(sizeof(*item), GFP_KERNEL);
5249         if (!item)
5250                 return -ENOMEM;
5251
5252         item->attrtype = attrtype;
5253         list_add_tail(&item->list, &fmsg->item_list);
5254
5255         return 0;
5256 }
5257
5258 int devlink_fmsg_obj_nest_start(struct devlink_fmsg *fmsg)
5259 {
5260         if (fmsg->putting_binary)
5261                 return -EINVAL;
5262
5263         return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_OBJ_NEST_START);
5264 }
5265 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_start);
5266
5267 static int devlink_fmsg_nest_end(struct devlink_fmsg *fmsg)
5268 {
5269         if (fmsg->putting_binary)
5270                 return -EINVAL;
5271
5272         return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_NEST_END);
5273 }
5274
5275 int devlink_fmsg_obj_nest_end(struct devlink_fmsg *fmsg)
5276 {
5277         if (fmsg->putting_binary)
5278                 return -EINVAL;
5279
5280         return devlink_fmsg_nest_end(fmsg);
5281 }
5282 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_end);
5283
5284 #define DEVLINK_FMSG_MAX_SIZE (GENLMSG_DEFAULT_SIZE - GENL_HDRLEN - NLA_HDRLEN)
5285
5286 static int devlink_fmsg_put_name(struct devlink_fmsg *fmsg, const char *name)
5287 {
5288         struct devlink_fmsg_item *item;
5289
5290         if (fmsg->putting_binary)
5291                 return -EINVAL;
5292
5293         if (strlen(name) + 1 > DEVLINK_FMSG_MAX_SIZE)
5294                 return -EMSGSIZE;
5295
5296         item = kzalloc(sizeof(*item) + strlen(name) + 1, GFP_KERNEL);
5297         if (!item)
5298                 return -ENOMEM;
5299
5300         item->nla_type = NLA_NUL_STRING;
5301         item->len = strlen(name) + 1;
5302         item->attrtype = DEVLINK_ATTR_FMSG_OBJ_NAME;
5303         memcpy(&item->value, name, item->len);
5304         list_add_tail(&item->list, &fmsg->item_list);
5305
5306         return 0;
5307 }
5308
5309 int devlink_fmsg_pair_nest_start(struct devlink_fmsg *fmsg, const char *name)
5310 {
5311         int err;
5312
5313         if (fmsg->putting_binary)
5314                 return -EINVAL;
5315
5316         err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_PAIR_NEST_START);
5317         if (err)
5318                 return err;
5319
5320         err = devlink_fmsg_put_name(fmsg, name);
5321         if (err)
5322                 return err;
5323
5324         return 0;
5325 }
5326 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_start);
5327
5328 int devlink_fmsg_pair_nest_end(struct devlink_fmsg *fmsg)
5329 {
5330         if (fmsg->putting_binary)
5331                 return -EINVAL;
5332
5333         return devlink_fmsg_nest_end(fmsg);
5334 }
5335 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_end);
5336
5337 int devlink_fmsg_arr_pair_nest_start(struct devlink_fmsg *fmsg,
5338                                      const char *name)
5339 {
5340         int err;
5341
5342         if (fmsg->putting_binary)
5343                 return -EINVAL;
5344
5345         err = devlink_fmsg_pair_nest_start(fmsg, name);
5346         if (err)
5347                 return err;
5348
5349         err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_ARR_NEST_START);
5350         if (err)
5351                 return err;
5352
5353         return 0;
5354 }
5355 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_start);
5356
5357 int devlink_fmsg_arr_pair_nest_end(struct devlink_fmsg *fmsg)
5358 {
5359         int err;
5360
5361         if (fmsg->putting_binary)
5362                 return -EINVAL;
5363
5364         err = devlink_fmsg_nest_end(fmsg);
5365         if (err)
5366                 return err;
5367
5368         err = devlink_fmsg_nest_end(fmsg);
5369         if (err)
5370                 return err;
5371
5372         return 0;
5373 }
5374 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_end);
5375
5376 int devlink_fmsg_binary_pair_nest_start(struct devlink_fmsg *fmsg,
5377                                         const char *name)
5378 {
5379         int err;
5380
5381         err = devlink_fmsg_arr_pair_nest_start(fmsg, name);
5382         if (err)
5383                 return err;
5384
5385         fmsg->putting_binary = true;
5386         return err;
5387 }
5388 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_start);
5389
5390 int devlink_fmsg_binary_pair_nest_end(struct devlink_fmsg *fmsg)
5391 {
5392         if (!fmsg->putting_binary)
5393                 return -EINVAL;
5394
5395         fmsg->putting_binary = false;
5396         return devlink_fmsg_arr_pair_nest_end(fmsg);
5397 }
5398 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_end);
5399
5400 static int devlink_fmsg_put_value(struct devlink_fmsg *fmsg,
5401                                   const void *value, u16 value_len,
5402                                   u8 value_nla_type)
5403 {
5404         struct devlink_fmsg_item *item;
5405
5406         if (value_len > DEVLINK_FMSG_MAX_SIZE)
5407                 return -EMSGSIZE;
5408
5409         item = kzalloc(sizeof(*item) + value_len, GFP_KERNEL);
5410         if (!item)
5411                 return -ENOMEM;
5412
5413         item->nla_type = value_nla_type;
5414         item->len = value_len;
5415         item->attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
5416         memcpy(&item->value, value, item->len);
5417         list_add_tail(&item->list, &fmsg->item_list);
5418
5419         return 0;
5420 }
5421
5422 int devlink_fmsg_bool_put(struct devlink_fmsg *fmsg, bool value)
5423 {
5424         if (fmsg->putting_binary)
5425                 return -EINVAL;
5426
5427         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_FLAG);
5428 }
5429 EXPORT_SYMBOL_GPL(devlink_fmsg_bool_put);
5430
5431 int devlink_fmsg_u8_put(struct devlink_fmsg *fmsg, u8 value)
5432 {
5433         if (fmsg->putting_binary)
5434                 return -EINVAL;
5435
5436         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U8);
5437 }
5438 EXPORT_SYMBOL_GPL(devlink_fmsg_u8_put);
5439
5440 int devlink_fmsg_u32_put(struct devlink_fmsg *fmsg, u32 value)
5441 {
5442         if (fmsg->putting_binary)
5443                 return -EINVAL;
5444
5445         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U32);
5446 }
5447 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_put);
5448
5449 int devlink_fmsg_u64_put(struct devlink_fmsg *fmsg, u64 value)
5450 {
5451         if (fmsg->putting_binary)
5452                 return -EINVAL;
5453
5454         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U64);
5455 }
5456 EXPORT_SYMBOL_GPL(devlink_fmsg_u64_put);
5457
5458 int devlink_fmsg_string_put(struct devlink_fmsg *fmsg, const char *value)
5459 {
5460         if (fmsg->putting_binary)
5461                 return -EINVAL;
5462
5463         return devlink_fmsg_put_value(fmsg, value, strlen(value) + 1,
5464                                       NLA_NUL_STRING);
5465 }
5466 EXPORT_SYMBOL_GPL(devlink_fmsg_string_put);
5467
5468 int devlink_fmsg_binary_put(struct devlink_fmsg *fmsg, const void *value,
5469                             u16 value_len)
5470 {
5471         if (!fmsg->putting_binary)
5472                 return -EINVAL;
5473
5474         return devlink_fmsg_put_value(fmsg, value, value_len, NLA_BINARY);
5475 }
5476 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_put);
5477
5478 int devlink_fmsg_bool_pair_put(struct devlink_fmsg *fmsg, const char *name,
5479                                bool value)
5480 {
5481         int err;
5482
5483         err = devlink_fmsg_pair_nest_start(fmsg, name);
5484         if (err)
5485                 return err;
5486
5487         err = devlink_fmsg_bool_put(fmsg, value);
5488         if (err)
5489                 return err;
5490
5491         err = devlink_fmsg_pair_nest_end(fmsg);
5492         if (err)
5493                 return err;
5494
5495         return 0;
5496 }
5497 EXPORT_SYMBOL_GPL(devlink_fmsg_bool_pair_put);
5498
5499 int devlink_fmsg_u8_pair_put(struct devlink_fmsg *fmsg, const char *name,
5500                              u8 value)
5501 {
5502         int err;
5503
5504         err = devlink_fmsg_pair_nest_start(fmsg, name);
5505         if (err)
5506                 return err;
5507
5508         err = devlink_fmsg_u8_put(fmsg, value);
5509         if (err)
5510                 return err;
5511
5512         err = devlink_fmsg_pair_nest_end(fmsg);
5513         if (err)
5514                 return err;
5515
5516         return 0;
5517 }
5518 EXPORT_SYMBOL_GPL(devlink_fmsg_u8_pair_put);
5519
5520 int devlink_fmsg_u32_pair_put(struct devlink_fmsg *fmsg, const char *name,
5521                               u32 value)
5522 {
5523         int err;
5524
5525         err = devlink_fmsg_pair_nest_start(fmsg, name);
5526         if (err)
5527                 return err;
5528
5529         err = devlink_fmsg_u32_put(fmsg, value);
5530         if (err)
5531                 return err;
5532
5533         err = devlink_fmsg_pair_nest_end(fmsg);
5534         if (err)
5535                 return err;
5536
5537         return 0;
5538 }
5539 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_pair_put);
5540
5541 int devlink_fmsg_u64_pair_put(struct devlink_fmsg *fmsg, const char *name,
5542                               u64 value)
5543 {
5544         int err;
5545
5546         err = devlink_fmsg_pair_nest_start(fmsg, name);
5547         if (err)
5548                 return err;
5549
5550         err = devlink_fmsg_u64_put(fmsg, value);
5551         if (err)
5552                 return err;
5553
5554         err = devlink_fmsg_pair_nest_end(fmsg);
5555         if (err)
5556                 return err;
5557
5558         return 0;
5559 }
5560 EXPORT_SYMBOL_GPL(devlink_fmsg_u64_pair_put);
5561
5562 int devlink_fmsg_string_pair_put(struct devlink_fmsg *fmsg, const char *name,
5563                                  const char *value)
5564 {
5565         int err;
5566
5567         err = devlink_fmsg_pair_nest_start(fmsg, name);
5568         if (err)
5569                 return err;
5570
5571         err = devlink_fmsg_string_put(fmsg, value);
5572         if (err)
5573                 return err;
5574
5575         err = devlink_fmsg_pair_nest_end(fmsg);
5576         if (err)
5577                 return err;
5578
5579         return 0;
5580 }
5581 EXPORT_SYMBOL_GPL(devlink_fmsg_string_pair_put);
5582
5583 int devlink_fmsg_binary_pair_put(struct devlink_fmsg *fmsg, const char *name,
5584                                  const void *value, u32 value_len)
5585 {
5586         u32 data_size;
5587         int end_err;
5588         u32 offset;
5589         int err;
5590
5591         err = devlink_fmsg_binary_pair_nest_start(fmsg, name);
5592         if (err)
5593                 return err;
5594
5595         for (offset = 0; offset < value_len; offset += data_size) {
5596                 data_size = value_len - offset;
5597                 if (data_size > DEVLINK_FMSG_MAX_SIZE)
5598                         data_size = DEVLINK_FMSG_MAX_SIZE;
5599                 err = devlink_fmsg_binary_put(fmsg, value + offset, data_size);
5600                 if (err)
5601                         break;
5602                 /* Exit from loop with a break (instead of
5603                  * return) to make sure putting_binary is turned off in
5604                  * devlink_fmsg_binary_pair_nest_end
5605                  */
5606         }
5607
5608         end_err = devlink_fmsg_binary_pair_nest_end(fmsg);
5609         if (end_err)
5610                 err = end_err;
5611
5612         return err;
5613 }
5614 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_put);
5615
5616 static int
5617 devlink_fmsg_item_fill_type(struct devlink_fmsg_item *msg, struct sk_buff *skb)
5618 {
5619         switch (msg->nla_type) {
5620         case NLA_FLAG:
5621         case NLA_U8:
5622         case NLA_U32:
5623         case NLA_U64:
5624         case NLA_NUL_STRING:
5625         case NLA_BINARY:
5626                 return nla_put_u8(skb, DEVLINK_ATTR_FMSG_OBJ_VALUE_TYPE,
5627                                   msg->nla_type);
5628         default:
5629                 return -EINVAL;
5630         }
5631 }
5632
5633 static int
5634 devlink_fmsg_item_fill_data(struct devlink_fmsg_item *msg, struct sk_buff *skb)
5635 {
5636         int attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
5637         u8 tmp;
5638
5639         switch (msg->nla_type) {
5640         case NLA_FLAG:
5641                 /* Always provide flag data, regardless of its value */
5642                 tmp = *(bool *) msg->value;
5643
5644                 return nla_put_u8(skb, attrtype, tmp);
5645         case NLA_U8:
5646                 return nla_put_u8(skb, attrtype, *(u8 *) msg->value);
5647         case NLA_U32:
5648                 return nla_put_u32(skb, attrtype, *(u32 *) msg->value);
5649         case NLA_U64:
5650                 return nla_put_u64_64bit(skb, attrtype, *(u64 *) msg->value,
5651                                          DEVLINK_ATTR_PAD);
5652         case NLA_NUL_STRING:
5653                 return nla_put_string(skb, attrtype, (char *) &msg->value);
5654         case NLA_BINARY:
5655                 return nla_put(skb, attrtype, msg->len, (void *) &msg->value);
5656         default:
5657                 return -EINVAL;
5658         }
5659 }
5660
5661 static int
5662 devlink_fmsg_prepare_skb(struct devlink_fmsg *fmsg, struct sk_buff *skb,
5663                          int *start)
5664 {
5665         struct devlink_fmsg_item *item;
5666         struct nlattr *fmsg_nlattr;
5667         int i = 0;
5668         int err;
5669
5670         fmsg_nlattr = nla_nest_start_noflag(skb, DEVLINK_ATTR_FMSG);
5671         if (!fmsg_nlattr)
5672                 return -EMSGSIZE;
5673
5674         list_for_each_entry(item, &fmsg->item_list, list) {
5675                 if (i < *start) {
5676                         i++;
5677                         continue;
5678                 }
5679
5680                 switch (item->attrtype) {
5681                 case DEVLINK_ATTR_FMSG_OBJ_NEST_START:
5682                 case DEVLINK_ATTR_FMSG_PAIR_NEST_START:
5683                 case DEVLINK_ATTR_FMSG_ARR_NEST_START:
5684                 case DEVLINK_ATTR_FMSG_NEST_END:
5685                         err = nla_put_flag(skb, item->attrtype);
5686                         break;
5687                 case DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA:
5688                         err = devlink_fmsg_item_fill_type(item, skb);
5689                         if (err)
5690                                 break;
5691                         err = devlink_fmsg_item_fill_data(item, skb);
5692                         break;
5693                 case DEVLINK_ATTR_FMSG_OBJ_NAME:
5694                         err = nla_put_string(skb, item->attrtype,
5695                                              (char *) &item->value);
5696                         break;
5697                 default:
5698                         err = -EINVAL;
5699                         break;
5700                 }
5701                 if (!err)
5702                         *start = ++i;
5703                 else
5704                         break;
5705         }
5706
5707         nla_nest_end(skb, fmsg_nlattr);
5708         return err;
5709 }
5710
5711 static int devlink_fmsg_snd(struct devlink_fmsg *fmsg,
5712                             struct genl_info *info,
5713                             enum devlink_command cmd, int flags)
5714 {
5715         struct nlmsghdr *nlh;
5716         struct sk_buff *skb;
5717         bool last = false;
5718         int index = 0;
5719         void *hdr;
5720         int err;
5721
5722         while (!last) {
5723                 int tmp_index = index;
5724
5725                 skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
5726                 if (!skb)
5727                         return -ENOMEM;
5728
5729                 hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
5730                                   &devlink_nl_family, flags | NLM_F_MULTI, cmd);
5731                 if (!hdr) {
5732                         err = -EMSGSIZE;
5733                         goto nla_put_failure;
5734                 }
5735
5736                 err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
5737                 if (!err)
5738                         last = true;
5739                 else if (err != -EMSGSIZE || tmp_index == index)
5740                         goto nla_put_failure;
5741
5742                 genlmsg_end(skb, hdr);
5743                 err = genlmsg_reply(skb, info);
5744                 if (err)
5745                         return err;
5746         }
5747
5748         skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
5749         if (!skb)
5750                 return -ENOMEM;
5751         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
5752                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
5753         if (!nlh) {
5754                 err = -EMSGSIZE;
5755                 goto nla_put_failure;
5756         }
5757
5758         return genlmsg_reply(skb, info);
5759
5760 nla_put_failure:
5761         nlmsg_free(skb);
5762         return err;
5763 }
5764
5765 static int devlink_fmsg_dumpit(struct devlink_fmsg *fmsg, struct sk_buff *skb,
5766                                struct netlink_callback *cb,
5767                                enum devlink_command cmd)
5768 {
5769         int index = cb->args[0];
5770         int tmp_index = index;
5771         void *hdr;
5772         int err;
5773
5774         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
5775                           &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI, cmd);
5776         if (!hdr) {
5777                 err = -EMSGSIZE;
5778                 goto nla_put_failure;
5779         }
5780
5781         err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
5782         if ((err && err != -EMSGSIZE) || tmp_index == index)
5783                 goto nla_put_failure;
5784
5785         cb->args[0] = index;
5786         genlmsg_end(skb, hdr);
5787         return skb->len;
5788
5789 nla_put_failure:
5790         genlmsg_cancel(skb, hdr);
5791         return err;
5792 }
5793
5794 struct devlink_health_reporter {
5795         struct list_head list;
5796         void *priv;
5797         const struct devlink_health_reporter_ops *ops;
5798         struct devlink *devlink;
5799         struct devlink_port *devlink_port;
5800         struct devlink_fmsg *dump_fmsg;
5801         struct mutex dump_lock; /* lock parallel read/write from dump buffers */
5802         u64 graceful_period;
5803         bool auto_recover;
5804         bool auto_dump;
5805         u8 health_state;
5806         u64 dump_ts;
5807         u64 dump_real_ts;
5808         u64 error_count;
5809         u64 recovery_count;
5810         u64 last_recovery_ts;
5811         refcount_t refcount;
5812 };
5813
5814 void *
5815 devlink_health_reporter_priv(struct devlink_health_reporter *reporter)
5816 {
5817         return reporter->priv;
5818 }
5819 EXPORT_SYMBOL_GPL(devlink_health_reporter_priv);
5820
5821 static struct devlink_health_reporter *
5822 __devlink_health_reporter_find_by_name(struct list_head *reporter_list,
5823                                        struct mutex *list_lock,
5824                                        const char *reporter_name)
5825 {
5826         struct devlink_health_reporter *reporter;
5827
5828         lockdep_assert_held(list_lock);
5829         list_for_each_entry(reporter, reporter_list, list)
5830                 if (!strcmp(reporter->ops->name, reporter_name))
5831                         return reporter;
5832         return NULL;
5833 }
5834
5835 static struct devlink_health_reporter *
5836 devlink_health_reporter_find_by_name(struct devlink *devlink,
5837                                      const char *reporter_name)
5838 {
5839         return __devlink_health_reporter_find_by_name(&devlink->reporter_list,
5840                                                       &devlink->reporters_lock,
5841                                                       reporter_name);
5842 }
5843
5844 static struct devlink_health_reporter *
5845 devlink_port_health_reporter_find_by_name(struct devlink_port *devlink_port,
5846                                           const char *reporter_name)
5847 {
5848         return __devlink_health_reporter_find_by_name(&devlink_port->reporter_list,
5849                                                       &devlink_port->reporters_lock,
5850                                                       reporter_name);
5851 }
5852
5853 static struct devlink_health_reporter *
5854 __devlink_health_reporter_create(struct devlink *devlink,
5855                                  const struct devlink_health_reporter_ops *ops,
5856                                  u64 graceful_period, void *priv)
5857 {
5858         struct devlink_health_reporter *reporter;
5859
5860         if (WARN_ON(graceful_period && !ops->recover))
5861                 return ERR_PTR(-EINVAL);
5862
5863         reporter = kzalloc(sizeof(*reporter), GFP_KERNEL);
5864         if (!reporter)
5865                 return ERR_PTR(-ENOMEM);
5866
5867         reporter->priv = priv;
5868         reporter->ops = ops;
5869         reporter->devlink = devlink;
5870         reporter->graceful_period = graceful_period;
5871         reporter->auto_recover = !!ops->recover;
5872         reporter->auto_dump = !!ops->dump;
5873         mutex_init(&reporter->dump_lock);
5874         refcount_set(&reporter->refcount, 1);
5875         return reporter;
5876 }
5877
5878 /**
5879  *      devlink_port_health_reporter_create - create devlink health reporter for
5880  *                                            specified port instance
5881  *
5882  *      @port: devlink_port which should contain the new reporter
5883  *      @ops: ops
5884  *      @graceful_period: to avoid recovery loops, in msecs
5885  *      @priv: priv
5886  */
5887 struct devlink_health_reporter *
5888 devlink_port_health_reporter_create(struct devlink_port *port,
5889                                     const struct devlink_health_reporter_ops *ops,
5890                                     u64 graceful_period, void *priv)
5891 {
5892         struct devlink_health_reporter *reporter;
5893
5894         mutex_lock(&port->reporters_lock);
5895         if (__devlink_health_reporter_find_by_name(&port->reporter_list,
5896                                                    &port->reporters_lock, ops->name)) {
5897                 reporter = ERR_PTR(-EEXIST);
5898                 goto unlock;
5899         }
5900
5901         reporter = __devlink_health_reporter_create(port->devlink, ops,
5902                                                     graceful_period, priv);
5903         if (IS_ERR(reporter))
5904                 goto unlock;
5905
5906         reporter->devlink_port = port;
5907         list_add_tail(&reporter->list, &port->reporter_list);
5908 unlock:
5909         mutex_unlock(&port->reporters_lock);
5910         return reporter;
5911 }
5912 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_create);
5913
5914 /**
5915  *      devlink_health_reporter_create - create devlink health reporter
5916  *
5917  *      @devlink: devlink
5918  *      @ops: ops
5919  *      @graceful_period: to avoid recovery loops, in msecs
5920  *      @priv: priv
5921  */
5922 struct devlink_health_reporter *
5923 devlink_health_reporter_create(struct devlink *devlink,
5924                                const struct devlink_health_reporter_ops *ops,
5925                                u64 graceful_period, void *priv)
5926 {
5927         struct devlink_health_reporter *reporter;
5928
5929         mutex_lock(&devlink->reporters_lock);
5930         if (devlink_health_reporter_find_by_name(devlink, ops->name)) {
5931                 reporter = ERR_PTR(-EEXIST);
5932                 goto unlock;
5933         }
5934
5935         reporter = __devlink_health_reporter_create(devlink, ops,
5936                                                     graceful_period, priv);
5937         if (IS_ERR(reporter))
5938                 goto unlock;
5939
5940         list_add_tail(&reporter->list, &devlink->reporter_list);
5941 unlock:
5942         mutex_unlock(&devlink->reporters_lock);
5943         return reporter;
5944 }
5945 EXPORT_SYMBOL_GPL(devlink_health_reporter_create);
5946
5947 static void
5948 devlink_health_reporter_free(struct devlink_health_reporter *reporter)
5949 {
5950         mutex_destroy(&reporter->dump_lock);
5951         if (reporter->dump_fmsg)
5952                 devlink_fmsg_free(reporter->dump_fmsg);
5953         kfree(reporter);
5954 }
5955
5956 static void
5957 devlink_health_reporter_put(struct devlink_health_reporter *reporter)
5958 {
5959         if (refcount_dec_and_test(&reporter->refcount))
5960                 devlink_health_reporter_free(reporter);
5961 }
5962
5963 static void
5964 __devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
5965 {
5966         list_del(&reporter->list);
5967         devlink_health_reporter_put(reporter);
5968 }
5969
5970 /**
5971  *      devlink_health_reporter_destroy - destroy devlink health reporter
5972  *
5973  *      @reporter: devlink health reporter to destroy
5974  */
5975 void
5976 devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
5977 {
5978         struct mutex *lock = &reporter->devlink->reporters_lock;
5979
5980         mutex_lock(lock);
5981         __devlink_health_reporter_destroy(reporter);
5982         mutex_unlock(lock);
5983 }
5984 EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);
5985
5986 /**
5987  *      devlink_port_health_reporter_destroy - destroy devlink port health reporter
5988  *
5989  *      @reporter: devlink health reporter to destroy
5990  */
5991 void
5992 devlink_port_health_reporter_destroy(struct devlink_health_reporter *reporter)
5993 {
5994         struct mutex *lock = &reporter->devlink_port->reporters_lock;
5995
5996         mutex_lock(lock);
5997         __devlink_health_reporter_destroy(reporter);
5998         mutex_unlock(lock);
5999 }
6000 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_destroy);
6001
6002 static int
6003 devlink_nl_health_reporter_fill(struct sk_buff *msg,
6004                                 struct devlink *devlink,
6005                                 struct devlink_health_reporter *reporter,
6006                                 enum devlink_command cmd, u32 portid,
6007                                 u32 seq, int flags)
6008 {
6009         struct nlattr *reporter_attr;
6010         void *hdr;
6011
6012         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
6013         if (!hdr)
6014                 return -EMSGSIZE;
6015
6016         if (devlink_nl_put_handle(msg, devlink))
6017                 goto genlmsg_cancel;
6018
6019         if (reporter->devlink_port) {
6020                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, reporter->devlink_port->index))
6021                         goto genlmsg_cancel;
6022         }
6023         reporter_attr = nla_nest_start_noflag(msg,
6024                                               DEVLINK_ATTR_HEALTH_REPORTER);
6025         if (!reporter_attr)
6026                 goto genlmsg_cancel;
6027         if (nla_put_string(msg, DEVLINK_ATTR_HEALTH_REPORTER_NAME,
6028                            reporter->ops->name))
6029                 goto reporter_nest_cancel;
6030         if (nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_STATE,
6031                        reporter->health_state))
6032                 goto reporter_nest_cancel;
6033         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_ERR_COUNT,
6034                               reporter->error_count, DEVLINK_ATTR_PAD))
6035                 goto reporter_nest_cancel;
6036         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_RECOVER_COUNT,
6037                               reporter->recovery_count, DEVLINK_ATTR_PAD))
6038                 goto reporter_nest_cancel;
6039         if (reporter->ops->recover &&
6040             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD,
6041                               reporter->graceful_period,
6042                               DEVLINK_ATTR_PAD))
6043                 goto reporter_nest_cancel;
6044         if (reporter->ops->recover &&
6045             nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER,
6046                        reporter->auto_recover))
6047                 goto reporter_nest_cancel;
6048         if (reporter->dump_fmsg &&
6049             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS,
6050                               jiffies_to_msecs(reporter->dump_ts),
6051                               DEVLINK_ATTR_PAD))
6052                 goto reporter_nest_cancel;
6053         if (reporter->dump_fmsg &&
6054             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS_NS,
6055                               reporter->dump_real_ts, DEVLINK_ATTR_PAD))
6056                 goto reporter_nest_cancel;
6057         if (reporter->ops->dump &&
6058             nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP,
6059                        reporter->auto_dump))
6060                 goto reporter_nest_cancel;
6061
6062         nla_nest_end(msg, reporter_attr);
6063         genlmsg_end(msg, hdr);
6064         return 0;
6065
6066 reporter_nest_cancel:
6067         nla_nest_end(msg, reporter_attr);
6068 genlmsg_cancel:
6069         genlmsg_cancel(msg, hdr);
6070         return -EMSGSIZE;
6071 }
6072
6073 static void devlink_recover_notify(struct devlink_health_reporter *reporter,
6074                                    enum devlink_command cmd)
6075 {
6076         struct sk_buff *msg;
6077         int err;
6078
6079         WARN_ON(cmd != DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
6080
6081         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
6082         if (!msg)
6083                 return;
6084
6085         err = devlink_nl_health_reporter_fill(msg, reporter->devlink,
6086                                               reporter, cmd, 0, 0, 0);
6087         if (err) {
6088                 nlmsg_free(msg);
6089                 return;
6090         }
6091
6092         genlmsg_multicast_netns(&devlink_nl_family,
6093                                 devlink_net(reporter->devlink),
6094                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
6095 }
6096
6097 void
6098 devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
6099 {
6100         reporter->recovery_count++;
6101         reporter->last_recovery_ts = jiffies;
6102 }
6103 EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
6104
6105 static int
6106 devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
6107                                 void *priv_ctx, struct netlink_ext_ack *extack)
6108 {
6109         int err;
6110
6111         if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
6112                 return 0;
6113
6114         if (!reporter->ops->recover)
6115                 return -EOPNOTSUPP;
6116
6117         err = reporter->ops->recover(reporter, priv_ctx, extack);
6118         if (err)
6119                 return err;
6120
6121         devlink_health_reporter_recovery_done(reporter);
6122         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
6123         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
6124
6125         return 0;
6126 }
6127
6128 static void
6129 devlink_health_dump_clear(struct devlink_health_reporter *reporter)
6130 {
6131         if (!reporter->dump_fmsg)
6132                 return;
6133         devlink_fmsg_free(reporter->dump_fmsg);
6134         reporter->dump_fmsg = NULL;
6135 }
6136
6137 static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
6138                                   void *priv_ctx,
6139                                   struct netlink_ext_ack *extack)
6140 {
6141         int err;
6142
6143         if (!reporter->ops->dump)
6144                 return 0;
6145
6146         if (reporter->dump_fmsg)
6147                 return 0;
6148
6149         reporter->dump_fmsg = devlink_fmsg_alloc();
6150         if (!reporter->dump_fmsg) {
6151                 err = -ENOMEM;
6152                 return err;
6153         }
6154
6155         err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
6156         if (err)
6157                 goto dump_err;
6158
6159         err = reporter->ops->dump(reporter, reporter->dump_fmsg,
6160                                   priv_ctx, extack);
6161         if (err)
6162                 goto dump_err;
6163
6164         err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
6165         if (err)
6166                 goto dump_err;
6167
6168         reporter->dump_ts = jiffies;
6169         reporter->dump_real_ts = ktime_get_real_ns();
6170
6171         return 0;
6172
6173 dump_err:
6174         devlink_health_dump_clear(reporter);
6175         return err;
6176 }
6177
6178 int devlink_health_report(struct devlink_health_reporter *reporter,
6179                           const char *msg, void *priv_ctx)
6180 {
6181         enum devlink_health_reporter_state prev_health_state;
6182         struct devlink *devlink = reporter->devlink;
6183         unsigned long recover_ts_threshold;
6184
6185         /* write a log message of the current error */
6186         WARN_ON(!msg);
6187         trace_devlink_health_report(devlink, reporter->ops->name, msg);
6188         reporter->error_count++;
6189         prev_health_state = reporter->health_state;
6190         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
6191         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
6192
6193         /* abort if the previous error wasn't recovered */
6194         recover_ts_threshold = reporter->last_recovery_ts +
6195                                msecs_to_jiffies(reporter->graceful_period);
6196         if (reporter->auto_recover &&
6197             (prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
6198              (reporter->last_recovery_ts && reporter->recovery_count &&
6199               time_is_after_jiffies(recover_ts_threshold)))) {
6200                 trace_devlink_health_recover_aborted(devlink,
6201                                                      reporter->ops->name,
6202                                                      reporter->health_state,
6203                                                      jiffies -
6204                                                      reporter->last_recovery_ts);
6205                 return -ECANCELED;
6206         }
6207
6208         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
6209
6210         if (reporter->auto_dump) {
6211                 mutex_lock(&reporter->dump_lock);
6212                 /* store current dump of current error, for later analysis */
6213                 devlink_health_do_dump(reporter, priv_ctx, NULL);
6214                 mutex_unlock(&reporter->dump_lock);
6215         }
6216
6217         if (reporter->auto_recover)
6218                 return devlink_health_reporter_recover(reporter,
6219                                                        priv_ctx, NULL);
6220
6221         return 0;
6222 }
6223 EXPORT_SYMBOL_GPL(devlink_health_report);
6224
6225 static struct devlink_health_reporter *
6226 devlink_health_reporter_get_from_attrs(struct devlink *devlink,
6227                                        struct nlattr **attrs)
6228 {
6229         struct devlink_health_reporter *reporter;
6230         struct devlink_port *devlink_port;
6231         char *reporter_name;
6232
6233         if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
6234                 return NULL;
6235
6236         reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
6237         devlink_port = devlink_port_get_from_attrs(devlink, attrs);
6238         if (IS_ERR(devlink_port)) {
6239                 mutex_lock(&devlink->reporters_lock);
6240                 reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
6241                 if (reporter)
6242                         refcount_inc(&reporter->refcount);
6243                 mutex_unlock(&devlink->reporters_lock);
6244         } else {
6245                 mutex_lock(&devlink_port->reporters_lock);
6246                 reporter = devlink_port_health_reporter_find_by_name(devlink_port, reporter_name);
6247                 if (reporter)
6248                         refcount_inc(&reporter->refcount);
6249                 mutex_unlock(&devlink_port->reporters_lock);
6250         }
6251
6252         return reporter;
6253 }
6254
6255 static struct devlink_health_reporter *
6256 devlink_health_reporter_get_from_info(struct devlink *devlink,
6257                                       struct genl_info *info)
6258 {
6259         return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
6260 }
6261
6262 static struct devlink_health_reporter *
6263 devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
6264 {
6265         const struct genl_dumpit_info *info = genl_dumpit_info(cb);
6266         struct devlink_health_reporter *reporter;
6267         struct nlattr **attrs = info->attrs;
6268         struct devlink *devlink;
6269
6270         mutex_lock(&devlink_mutex);
6271         devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
6272         if (IS_ERR(devlink))
6273                 goto unlock;
6274
6275         reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
6276         mutex_unlock(&devlink_mutex);
6277         return reporter;
6278 unlock:
6279         mutex_unlock(&devlink_mutex);
6280         return NULL;
6281 }
6282
6283 void
6284 devlink_health_reporter_state_update(struct devlink_health_reporter *reporter,
6285                                      enum devlink_health_reporter_state state)
6286 {
6287         if (WARN_ON(state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY &&
6288                     state != DEVLINK_HEALTH_REPORTER_STATE_ERROR))
6289                 return;
6290
6291         if (reporter->health_state == state)
6292                 return;
6293
6294         reporter->health_state = state;
6295         trace_devlink_health_reporter_state_update(reporter->devlink,
6296                                                    reporter->ops->name, state);
6297         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
6298 }
6299 EXPORT_SYMBOL_GPL(devlink_health_reporter_state_update);
6300
6301 static int devlink_nl_cmd_health_reporter_get_doit(struct sk_buff *skb,
6302                                                    struct genl_info *info)
6303 {
6304         struct devlink *devlink = info->user_ptr[0];
6305         struct devlink_health_reporter *reporter;
6306         struct sk_buff *msg;
6307         int err;
6308
6309         reporter = devlink_health_reporter_get_from_info(devlink, info);
6310         if (!reporter)
6311                 return -EINVAL;
6312
6313         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
6314         if (!msg) {
6315                 err = -ENOMEM;
6316                 goto out;
6317         }
6318
6319         err = devlink_nl_health_reporter_fill(msg, devlink, reporter,
6320                                               DEVLINK_CMD_HEALTH_REPORTER_GET,
6321                                               info->snd_portid, info->snd_seq,
6322                                               0);
6323         if (err) {
6324                 nlmsg_free(msg);
6325                 goto out;
6326         }
6327
6328         err = genlmsg_reply(msg, info);
6329 out:
6330         devlink_health_reporter_put(reporter);
6331         return err;
6332 }
6333
6334 static int
6335 devlink_nl_cmd_health_reporter_get_dumpit(struct sk_buff *msg,
6336                                           struct netlink_callback *cb)
6337 {
6338         struct devlink_health_reporter *reporter;
6339         struct devlink_port *port;
6340         struct devlink *devlink;
6341         int start = cb->args[0];
6342         int idx = 0;
6343         int err;
6344
6345         mutex_lock(&devlink_mutex);
6346         list_for_each_entry(devlink, &devlink_list, list) {
6347                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
6348                         continue;
6349                 mutex_lock(&devlink->reporters_lock);
6350                 list_for_each_entry(reporter, &devlink->reporter_list,
6351                                     list) {
6352                         if (idx < start) {
6353                                 idx++;
6354                                 continue;
6355                         }
6356                         err = devlink_nl_health_reporter_fill(msg, devlink,
6357                                                               reporter,
6358                                                               DEVLINK_CMD_HEALTH_REPORTER_GET,
6359                                                               NETLINK_CB(cb->skb).portid,
6360                                                               cb->nlh->nlmsg_seq,
6361                                                               NLM_F_MULTI);
6362                         if (err) {
6363                                 mutex_unlock(&devlink->reporters_lock);
6364                                 goto out;
6365                         }
6366                         idx++;
6367                 }
6368                 mutex_unlock(&devlink->reporters_lock);
6369         }
6370
6371         list_for_each_entry(devlink, &devlink_list, list) {
6372                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
6373                         continue;
6374                 mutex_lock(&devlink->lock);
6375                 list_for_each_entry(port, &devlink->port_list, list) {
6376                         mutex_lock(&port->reporters_lock);
6377                         list_for_each_entry(reporter, &port->reporter_list, list) {
6378                                 if (idx < start) {
6379                                         idx++;
6380                                         continue;
6381                                 }
6382                                 err = devlink_nl_health_reporter_fill(msg, devlink, reporter,
6383                                                                       DEVLINK_CMD_HEALTH_REPORTER_GET,
6384                                                                       NETLINK_CB(cb->skb).portid,
6385                                                                       cb->nlh->nlmsg_seq,
6386                                                                       NLM_F_MULTI);
6387                                 if (err) {
6388                                         mutex_unlock(&port->reporters_lock);
6389                                         mutex_unlock(&devlink->lock);
6390                                         goto out;
6391                                 }
6392                                 idx++;
6393                         }
6394                         mutex_unlock(&port->reporters_lock);
6395                 }
6396                 mutex_unlock(&devlink->lock);
6397         }
6398 out:
6399         mutex_unlock(&devlink_mutex);
6400
6401         cb->args[0] = idx;
6402         return msg->len;
6403 }
6404
6405 static int
6406 devlink_nl_cmd_health_reporter_set_doit(struct sk_buff *skb,
6407                                         struct genl_info *info)
6408 {
6409         struct devlink *devlink = info->user_ptr[0];
6410         struct devlink_health_reporter *reporter;
6411         int err;
6412
6413         reporter = devlink_health_reporter_get_from_info(devlink, info);
6414         if (!reporter)
6415                 return -EINVAL;
6416
6417         if (!reporter->ops->recover &&
6418             (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] ||
6419              info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])) {
6420                 err = -EOPNOTSUPP;
6421                 goto out;
6422         }
6423         if (!reporter->ops->dump &&
6424             info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]) {
6425                 err = -EOPNOTSUPP;
6426                 goto out;
6427         }
6428
6429         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD])
6430                 reporter->graceful_period =
6431                         nla_get_u64(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD]);
6432
6433         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])
6434                 reporter->auto_recover =
6435                         nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER]);
6436
6437         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP])
6438                 reporter->auto_dump =
6439                 nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]);
6440
6441         devlink_health_reporter_put(reporter);
6442         return 0;
6443 out:
6444         devlink_health_reporter_put(reporter);
6445         return err;
6446 }
6447
6448 static int devlink_nl_cmd_health_reporter_recover_doit(struct sk_buff *skb,
6449                                                        struct genl_info *info)
6450 {
6451         struct devlink *devlink = info->user_ptr[0];
6452         struct devlink_health_reporter *reporter;
6453         int err;
6454
6455         reporter = devlink_health_reporter_get_from_info(devlink, info);
6456         if (!reporter)
6457                 return -EINVAL;
6458
6459         err = devlink_health_reporter_recover(reporter, NULL, info->extack);
6460
6461         devlink_health_reporter_put(reporter);
6462         return err;
6463 }
6464
6465 static int devlink_nl_cmd_health_reporter_diagnose_doit(struct sk_buff *skb,
6466                                                         struct genl_info *info)
6467 {
6468         struct devlink *devlink = info->user_ptr[0];
6469         struct devlink_health_reporter *reporter;
6470         struct devlink_fmsg *fmsg;
6471         int err;
6472
6473         reporter = devlink_health_reporter_get_from_info(devlink, info);
6474         if (!reporter)
6475                 return -EINVAL;
6476
6477         if (!reporter->ops->diagnose) {
6478                 devlink_health_reporter_put(reporter);
6479                 return -EOPNOTSUPP;
6480         }
6481
6482         fmsg = devlink_fmsg_alloc();
6483         if (!fmsg) {
6484                 devlink_health_reporter_put(reporter);
6485                 return -ENOMEM;
6486         }
6487
6488         err = devlink_fmsg_obj_nest_start(fmsg);
6489         if (err)
6490                 goto out;
6491
6492         err = reporter->ops->diagnose(reporter, fmsg, info->extack);
6493         if (err)
6494                 goto out;
6495
6496         err = devlink_fmsg_obj_nest_end(fmsg);
6497         if (err)
6498                 goto out;
6499
6500         err = devlink_fmsg_snd(fmsg, info,
6501                                DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE, 0);
6502
6503 out:
6504         devlink_fmsg_free(fmsg);
6505         devlink_health_reporter_put(reporter);
6506         return err;
6507 }
6508
6509 static int
6510 devlink_nl_cmd_health_reporter_dump_get_dumpit(struct sk_buff *skb,
6511                                                struct netlink_callback *cb)
6512 {
6513         struct devlink_health_reporter *reporter;
6514         u64 start = cb->args[0];
6515         int err;
6516
6517         reporter = devlink_health_reporter_get_from_cb(cb);
6518         if (!reporter)
6519                 return -EINVAL;
6520
6521         if (!reporter->ops->dump) {
6522                 err = -EOPNOTSUPP;
6523                 goto out;
6524         }
6525         mutex_lock(&reporter->dump_lock);
6526         if (!start) {
6527                 err = devlink_health_do_dump(reporter, NULL, cb->extack);
6528                 if (err)
6529                         goto unlock;
6530                 cb->args[1] = reporter->dump_ts;
6531         }
6532         if (!reporter->dump_fmsg || cb->args[1] != reporter->dump_ts) {
6533                 NL_SET_ERR_MSG_MOD(cb->extack, "Dump trampled, please retry");
6534                 err = -EAGAIN;
6535                 goto unlock;
6536         }
6537
6538         err = devlink_fmsg_dumpit(reporter->dump_fmsg, skb, cb,
6539                                   DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET);
6540 unlock:
6541         mutex_unlock(&reporter->dump_lock);
6542 out:
6543         devlink_health_reporter_put(reporter);
6544         return err;
6545 }
6546
6547 static int
6548 devlink_nl_cmd_health_reporter_dump_clear_doit(struct sk_buff *skb,
6549                                                struct genl_info *info)
6550 {
6551         struct devlink *devlink = info->user_ptr[0];
6552         struct devlink_health_reporter *reporter;
6553
6554         reporter = devlink_health_reporter_get_from_info(devlink, info);
6555         if (!reporter)
6556                 return -EINVAL;
6557
6558         if (!reporter->ops->dump) {
6559                 devlink_health_reporter_put(reporter);
6560                 return -EOPNOTSUPP;
6561         }
6562
6563         mutex_lock(&reporter->dump_lock);
6564         devlink_health_dump_clear(reporter);
6565         mutex_unlock(&reporter->dump_lock);
6566         devlink_health_reporter_put(reporter);
6567         return 0;
6568 }
6569
6570 static int devlink_nl_cmd_health_reporter_test_doit(struct sk_buff *skb,
6571                                                     struct genl_info *info)
6572 {
6573         struct devlink *devlink = info->user_ptr[0];
6574         struct devlink_health_reporter *reporter;
6575         int err;
6576
6577         reporter = devlink_health_reporter_get_from_info(devlink, info);
6578         if (!reporter)
6579                 return -EINVAL;
6580
6581         if (!reporter->ops->test) {
6582                 devlink_health_reporter_put(reporter);
6583                 return -EOPNOTSUPP;
6584         }
6585
6586         err = reporter->ops->test(reporter, info->extack);
6587
6588         devlink_health_reporter_put(reporter);
6589         return err;
6590 }
6591
6592 struct devlink_stats {
6593         u64 rx_bytes;
6594         u64 rx_packets;
6595         struct u64_stats_sync syncp;
6596 };
6597
6598 /**
6599  * struct devlink_trap_policer_item - Packet trap policer attributes.
6600  * @policer: Immutable packet trap policer attributes.
6601  * @rate: Rate in packets / sec.
6602  * @burst: Burst size in packets.
6603  * @list: trap_policer_list member.
6604  *
6605  * Describes packet trap policer attributes. Created by devlink during trap
6606  * policer registration.
6607  */
6608 struct devlink_trap_policer_item {
6609         const struct devlink_trap_policer *policer;
6610         u64 rate;
6611         u64 burst;
6612         struct list_head list;
6613 };
6614
6615 /**
6616  * struct devlink_trap_group_item - Packet trap group attributes.
6617  * @group: Immutable packet trap group attributes.
6618  * @policer_item: Associated policer item. Can be NULL.
6619  * @list: trap_group_list member.
6620  * @stats: Trap group statistics.
6621  *
6622  * Describes packet trap group attributes. Created by devlink during trap
6623  * group registration.
6624  */
6625 struct devlink_trap_group_item {
6626         const struct devlink_trap_group *group;
6627         struct devlink_trap_policer_item *policer_item;
6628         struct list_head list;
6629         struct devlink_stats __percpu *stats;
6630 };
6631
6632 /**
6633  * struct devlink_trap_item - Packet trap attributes.
6634  * @trap: Immutable packet trap attributes.
6635  * @group_item: Associated group item.
6636  * @list: trap_list member.
6637  * @action: Trap action.
6638  * @stats: Trap statistics.
6639  * @priv: Driver private information.
6640  *
6641  * Describes both mutable and immutable packet trap attributes. Created by
6642  * devlink during trap registration and used for all trap related operations.
6643  */
6644 struct devlink_trap_item {
6645         const struct devlink_trap *trap;
6646         struct devlink_trap_group_item *group_item;
6647         struct list_head list;
6648         enum devlink_trap_action action;
6649         struct devlink_stats __percpu *stats;
6650         void *priv;
6651 };
6652
6653 static struct devlink_trap_policer_item *
6654 devlink_trap_policer_item_lookup(struct devlink *devlink, u32 id)
6655 {
6656         struct devlink_trap_policer_item *policer_item;
6657
6658         list_for_each_entry(policer_item, &devlink->trap_policer_list, list) {
6659                 if (policer_item->policer->id == id)
6660                         return policer_item;
6661         }
6662
6663         return NULL;
6664 }
6665
6666 static struct devlink_trap_item *
6667 devlink_trap_item_lookup(struct devlink *devlink, const char *name)
6668 {
6669         struct devlink_trap_item *trap_item;
6670
6671         list_for_each_entry(trap_item, &devlink->trap_list, list) {
6672                 if (!strcmp(trap_item->trap->name, name))
6673                         return trap_item;
6674         }
6675
6676         return NULL;
6677 }
6678
6679 static struct devlink_trap_item *
6680 devlink_trap_item_get_from_info(struct devlink *devlink,
6681                                 struct genl_info *info)
6682 {
6683         struct nlattr *attr;
6684
6685         if (!info->attrs[DEVLINK_ATTR_TRAP_NAME])
6686                 return NULL;
6687         attr = info->attrs[DEVLINK_ATTR_TRAP_NAME];
6688
6689         return devlink_trap_item_lookup(devlink, nla_data(attr));
6690 }
6691
6692 static int
6693 devlink_trap_action_get_from_info(struct genl_info *info,
6694                                   enum devlink_trap_action *p_trap_action)
6695 {
6696         u8 val;
6697
6698         val = nla_get_u8(info->attrs[DEVLINK_ATTR_TRAP_ACTION]);
6699         switch (val) {
6700         case DEVLINK_TRAP_ACTION_DROP:
6701         case DEVLINK_TRAP_ACTION_TRAP:
6702         case DEVLINK_TRAP_ACTION_MIRROR:
6703                 *p_trap_action = val;
6704                 break;
6705         default:
6706                 return -EINVAL;
6707         }
6708
6709         return 0;
6710 }
6711
6712 static int devlink_trap_metadata_put(struct sk_buff *msg,
6713                                      const struct devlink_trap *trap)
6714 {
6715         struct nlattr *attr;
6716
6717         attr = nla_nest_start(msg, DEVLINK_ATTR_TRAP_METADATA);
6718         if (!attr)
6719                 return -EMSGSIZE;
6720
6721         if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_IN_PORT) &&
6722             nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_IN_PORT))
6723                 goto nla_put_failure;
6724         if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_FA_COOKIE) &&
6725             nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_FA_COOKIE))
6726                 goto nla_put_failure;
6727
6728         nla_nest_end(msg, attr);
6729
6730         return 0;
6731
6732 nla_put_failure:
6733         nla_nest_cancel(msg, attr);
6734         return -EMSGSIZE;
6735 }
6736
6737 static void devlink_trap_stats_read(struct devlink_stats __percpu *trap_stats,
6738                                     struct devlink_stats *stats)
6739 {
6740         int i;
6741
6742         memset(stats, 0, sizeof(*stats));
6743         for_each_possible_cpu(i) {
6744                 struct devlink_stats *cpu_stats;
6745                 u64 rx_packets, rx_bytes;
6746                 unsigned int start;
6747
6748                 cpu_stats = per_cpu_ptr(trap_stats, i);
6749                 do {
6750                         start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
6751                         rx_packets = cpu_stats->rx_packets;
6752                         rx_bytes = cpu_stats->rx_bytes;
6753                 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
6754
6755                 stats->rx_packets += rx_packets;
6756                 stats->rx_bytes += rx_bytes;
6757         }
6758 }
6759
6760 static int devlink_trap_stats_put(struct sk_buff *msg,
6761                                   struct devlink_stats __percpu *trap_stats)
6762 {
6763         struct devlink_stats stats;
6764         struct nlattr *attr;
6765
6766         devlink_trap_stats_read(trap_stats, &stats);
6767
6768         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
6769         if (!attr)
6770                 return -EMSGSIZE;
6771
6772         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
6773                               stats.rx_packets, DEVLINK_ATTR_PAD))
6774                 goto nla_put_failure;
6775
6776         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
6777                               stats.rx_bytes, DEVLINK_ATTR_PAD))
6778                 goto nla_put_failure;
6779
6780         nla_nest_end(msg, attr);
6781
6782         return 0;
6783
6784 nla_put_failure:
6785         nla_nest_cancel(msg, attr);
6786         return -EMSGSIZE;
6787 }
6788
6789 static int devlink_nl_trap_fill(struct sk_buff *msg, struct devlink *devlink,
6790                                 const struct devlink_trap_item *trap_item,
6791                                 enum devlink_command cmd, u32 portid, u32 seq,
6792                                 int flags)
6793 {
6794         struct devlink_trap_group_item *group_item = trap_item->group_item;
6795         void *hdr;
6796         int err;
6797
6798         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
6799         if (!hdr)
6800                 return -EMSGSIZE;
6801
6802         if (devlink_nl_put_handle(msg, devlink))
6803                 goto nla_put_failure;
6804
6805         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
6806                            group_item->group->name))
6807                 goto nla_put_failure;
6808
6809         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_NAME, trap_item->trap->name))
6810                 goto nla_put_failure;
6811
6812         if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_TYPE, trap_item->trap->type))
6813                 goto nla_put_failure;
6814
6815         if (trap_item->trap->generic &&
6816             nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
6817                 goto nla_put_failure;
6818
6819         if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_ACTION, trap_item->action))
6820                 goto nla_put_failure;
6821
6822         err = devlink_trap_metadata_put(msg, trap_item->trap);
6823         if (err)
6824                 goto nla_put_failure;
6825
6826         err = devlink_trap_stats_put(msg, trap_item->stats);
6827         if (err)
6828                 goto nla_put_failure;
6829
6830         genlmsg_end(msg, hdr);
6831
6832         return 0;
6833
6834 nla_put_failure:
6835         genlmsg_cancel(msg, hdr);
6836         return -EMSGSIZE;
6837 }
6838
6839 static int devlink_nl_cmd_trap_get_doit(struct sk_buff *skb,
6840                                         struct genl_info *info)
6841 {
6842         struct netlink_ext_ack *extack = info->extack;
6843         struct devlink *devlink = info->user_ptr[0];
6844         struct devlink_trap_item *trap_item;
6845         struct sk_buff *msg;
6846         int err;
6847
6848         if (list_empty(&devlink->trap_list))
6849                 return -EOPNOTSUPP;
6850
6851         trap_item = devlink_trap_item_get_from_info(devlink, info);
6852         if (!trap_item) {
6853                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
6854                 return -ENOENT;
6855         }
6856
6857         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
6858         if (!msg)
6859                 return -ENOMEM;
6860
6861         err = devlink_nl_trap_fill(msg, devlink, trap_item,
6862                                    DEVLINK_CMD_TRAP_NEW, info->snd_portid,
6863                                    info->snd_seq, 0);
6864         if (err)
6865                 goto err_trap_fill;
6866
6867         return genlmsg_reply(msg, info);
6868
6869 err_trap_fill:
6870         nlmsg_free(msg);
6871         return err;
6872 }
6873
6874 static int devlink_nl_cmd_trap_get_dumpit(struct sk_buff *msg,
6875                                           struct netlink_callback *cb)
6876 {
6877         struct devlink_trap_item *trap_item;
6878         struct devlink *devlink;
6879         int start = cb->args[0];
6880         int idx = 0;
6881         int err;
6882
6883         mutex_lock(&devlink_mutex);
6884         list_for_each_entry(devlink, &devlink_list, list) {
6885                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
6886                         continue;
6887                 mutex_lock(&devlink->lock);
6888                 list_for_each_entry(trap_item, &devlink->trap_list, list) {
6889                         if (idx < start) {
6890                                 idx++;
6891                                 continue;
6892                         }
6893                         err = devlink_nl_trap_fill(msg, devlink, trap_item,
6894                                                    DEVLINK_CMD_TRAP_NEW,
6895                                                    NETLINK_CB(cb->skb).portid,
6896                                                    cb->nlh->nlmsg_seq,
6897                                                    NLM_F_MULTI);
6898                         if (err) {
6899                                 mutex_unlock(&devlink->lock);
6900                                 goto out;
6901                         }
6902                         idx++;
6903                 }
6904                 mutex_unlock(&devlink->lock);
6905         }
6906 out:
6907         mutex_unlock(&devlink_mutex);
6908
6909         cb->args[0] = idx;
6910         return msg->len;
6911 }
6912
6913 static int __devlink_trap_action_set(struct devlink *devlink,
6914                                      struct devlink_trap_item *trap_item,
6915                                      enum devlink_trap_action trap_action,
6916                                      struct netlink_ext_ack *extack)
6917 {
6918         int err;
6919
6920         if (trap_item->action != trap_action &&
6921             trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP) {
6922                 NL_SET_ERR_MSG_MOD(extack, "Cannot change action of non-drop traps. Skipping");
6923                 return 0;
6924         }
6925
6926         err = devlink->ops->trap_action_set(devlink, trap_item->trap,
6927                                             trap_action, extack);
6928         if (err)
6929                 return err;
6930
6931         trap_item->action = trap_action;
6932
6933         return 0;
6934 }
6935
6936 static int devlink_trap_action_set(struct devlink *devlink,
6937                                    struct devlink_trap_item *trap_item,
6938                                    struct genl_info *info)
6939 {
6940         enum devlink_trap_action trap_action;
6941         int err;
6942
6943         if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
6944                 return 0;
6945
6946         err = devlink_trap_action_get_from_info(info, &trap_action);
6947         if (err) {
6948                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
6949                 return -EINVAL;
6950         }
6951
6952         return __devlink_trap_action_set(devlink, trap_item, trap_action,
6953                                          info->extack);
6954 }
6955
6956 static int devlink_nl_cmd_trap_set_doit(struct sk_buff *skb,
6957                                         struct genl_info *info)
6958 {
6959         struct netlink_ext_ack *extack = info->extack;
6960         struct devlink *devlink = info->user_ptr[0];
6961         struct devlink_trap_item *trap_item;
6962         int err;
6963
6964         if (list_empty(&devlink->trap_list))
6965                 return -EOPNOTSUPP;
6966
6967         trap_item = devlink_trap_item_get_from_info(devlink, info);
6968         if (!trap_item) {
6969                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
6970                 return -ENOENT;
6971         }
6972
6973         err = devlink_trap_action_set(devlink, trap_item, info);
6974         if (err)
6975                 return err;
6976
6977         return 0;
6978 }
6979
6980 static struct devlink_trap_group_item *
6981 devlink_trap_group_item_lookup(struct devlink *devlink, const char *name)
6982 {
6983         struct devlink_trap_group_item *group_item;
6984
6985         list_for_each_entry(group_item, &devlink->trap_group_list, list) {
6986                 if (!strcmp(group_item->group->name, name))
6987                         return group_item;
6988         }
6989
6990         return NULL;
6991 }
6992
6993 static struct devlink_trap_group_item *
6994 devlink_trap_group_item_lookup_by_id(struct devlink *devlink, u16 id)
6995 {
6996         struct devlink_trap_group_item *group_item;
6997
6998         list_for_each_entry(group_item, &devlink->trap_group_list, list) {
6999                 if (group_item->group->id == id)
7000                         return group_item;
7001         }
7002
7003         return NULL;
7004 }
7005
7006 static struct devlink_trap_group_item *
7007 devlink_trap_group_item_get_from_info(struct devlink *devlink,
7008                                       struct genl_info *info)
7009 {
7010         char *name;
7011
7012         if (!info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME])
7013                 return NULL;
7014         name = nla_data(info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME]);
7015
7016         return devlink_trap_group_item_lookup(devlink, name);
7017 }
7018
7019 static int
7020 devlink_nl_trap_group_fill(struct sk_buff *msg, struct devlink *devlink,
7021                            const struct devlink_trap_group_item *group_item,
7022                            enum devlink_command cmd, u32 portid, u32 seq,
7023                            int flags)
7024 {
7025         void *hdr;
7026         int err;
7027
7028         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
7029         if (!hdr)
7030                 return -EMSGSIZE;
7031
7032         if (devlink_nl_put_handle(msg, devlink))
7033                 goto nla_put_failure;
7034
7035         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
7036                            group_item->group->name))
7037                 goto nla_put_failure;
7038
7039         if (group_item->group->generic &&
7040             nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
7041                 goto nla_put_failure;
7042
7043         if (group_item->policer_item &&
7044             nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
7045                         group_item->policer_item->policer->id))
7046                 goto nla_put_failure;
7047
7048         err = devlink_trap_stats_put(msg, group_item->stats);
7049         if (err)
7050                 goto nla_put_failure;
7051
7052         genlmsg_end(msg, hdr);
7053
7054         return 0;
7055
7056 nla_put_failure:
7057         genlmsg_cancel(msg, hdr);
7058         return -EMSGSIZE;
7059 }
7060
7061 static int devlink_nl_cmd_trap_group_get_doit(struct sk_buff *skb,
7062                                               struct genl_info *info)
7063 {
7064         struct netlink_ext_ack *extack = info->extack;
7065         struct devlink *devlink = info->user_ptr[0];
7066         struct devlink_trap_group_item *group_item;
7067         struct sk_buff *msg;
7068         int err;
7069
7070         if (list_empty(&devlink->trap_group_list))
7071                 return -EOPNOTSUPP;
7072
7073         group_item = devlink_trap_group_item_get_from_info(devlink, info);
7074         if (!group_item) {
7075                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
7076                 return -ENOENT;
7077         }
7078
7079         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7080         if (!msg)
7081                 return -ENOMEM;
7082
7083         err = devlink_nl_trap_group_fill(msg, devlink, group_item,
7084                                          DEVLINK_CMD_TRAP_GROUP_NEW,
7085                                          info->snd_portid, info->snd_seq, 0);
7086         if (err)
7087                 goto err_trap_group_fill;
7088
7089         return genlmsg_reply(msg, info);
7090
7091 err_trap_group_fill:
7092         nlmsg_free(msg);
7093         return err;
7094 }
7095
7096 static int devlink_nl_cmd_trap_group_get_dumpit(struct sk_buff *msg,
7097                                                 struct netlink_callback *cb)
7098 {
7099         enum devlink_command cmd = DEVLINK_CMD_TRAP_GROUP_NEW;
7100         struct devlink_trap_group_item *group_item;
7101         u32 portid = NETLINK_CB(cb->skb).portid;
7102         struct devlink *devlink;
7103         int start = cb->args[0];
7104         int idx = 0;
7105         int err;
7106
7107         mutex_lock(&devlink_mutex);
7108         list_for_each_entry(devlink, &devlink_list, list) {
7109                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7110                         continue;
7111                 mutex_lock(&devlink->lock);
7112                 list_for_each_entry(group_item, &devlink->trap_group_list,
7113                                     list) {
7114                         if (idx < start) {
7115                                 idx++;
7116                                 continue;
7117                         }
7118                         err = devlink_nl_trap_group_fill(msg, devlink,
7119                                                          group_item, cmd,
7120                                                          portid,
7121                                                          cb->nlh->nlmsg_seq,
7122                                                          NLM_F_MULTI);
7123                         if (err) {
7124                                 mutex_unlock(&devlink->lock);
7125                                 goto out;
7126                         }
7127                         idx++;
7128                 }
7129                 mutex_unlock(&devlink->lock);
7130         }
7131 out:
7132         mutex_unlock(&devlink_mutex);
7133
7134         cb->args[0] = idx;
7135         return msg->len;
7136 }
7137
7138 static int
7139 __devlink_trap_group_action_set(struct devlink *devlink,
7140                                 struct devlink_trap_group_item *group_item,
7141                                 enum devlink_trap_action trap_action,
7142                                 struct netlink_ext_ack *extack)
7143 {
7144         const char *group_name = group_item->group->name;
7145         struct devlink_trap_item *trap_item;
7146         int err;
7147
7148         if (devlink->ops->trap_group_action_set) {
7149                 err = devlink->ops->trap_group_action_set(devlink, group_item->group,
7150                                                           trap_action, extack);
7151                 if (err)
7152                         return err;
7153
7154                 list_for_each_entry(trap_item, &devlink->trap_list, list) {
7155                         if (strcmp(trap_item->group_item->group->name, group_name))
7156                                 continue;
7157                         if (trap_item->action != trap_action &&
7158                             trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP)
7159                                 continue;
7160                         trap_item->action = trap_action;
7161                 }
7162
7163                 return 0;
7164         }
7165
7166         list_for_each_entry(trap_item, &devlink->trap_list, list) {
7167                 if (strcmp(trap_item->group_item->group->name, group_name))
7168                         continue;
7169                 err = __devlink_trap_action_set(devlink, trap_item,
7170                                                 trap_action, extack);
7171                 if (err)
7172                         return err;
7173         }
7174
7175         return 0;
7176 }
7177
7178 static int
7179 devlink_trap_group_action_set(struct devlink *devlink,
7180                               struct devlink_trap_group_item *group_item,
7181                               struct genl_info *info, bool *p_modified)
7182 {
7183         enum devlink_trap_action trap_action;
7184         int err;
7185
7186         if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
7187                 return 0;
7188
7189         err = devlink_trap_action_get_from_info(info, &trap_action);
7190         if (err) {
7191                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
7192                 return -EINVAL;
7193         }
7194
7195         err = __devlink_trap_group_action_set(devlink, group_item, trap_action,
7196                                               info->extack);
7197         if (err)
7198                 return err;
7199
7200         *p_modified = true;
7201
7202         return 0;
7203 }
7204
7205 static int devlink_trap_group_set(struct devlink *devlink,
7206                                   struct devlink_trap_group_item *group_item,
7207                                   struct genl_info *info)
7208 {
7209         struct devlink_trap_policer_item *policer_item;
7210         struct netlink_ext_ack *extack = info->extack;
7211         const struct devlink_trap_policer *policer;
7212         struct nlattr **attrs = info->attrs;
7213         int err;
7214
7215         if (!attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
7216                 return 0;
7217
7218         if (!devlink->ops->trap_group_set)
7219                 return -EOPNOTSUPP;
7220
7221         policer_item = group_item->policer_item;
7222         if (attrs[DEVLINK_ATTR_TRAP_POLICER_ID]) {
7223                 u32 policer_id;
7224
7225                 policer_id = nla_get_u32(attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
7226                 policer_item = devlink_trap_policer_item_lookup(devlink,
7227                                                                 policer_id);
7228                 if (policer_id && !policer_item) {
7229                         NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
7230                         return -ENOENT;
7231                 }
7232         }
7233         policer = policer_item ? policer_item->policer : NULL;
7234
7235         err = devlink->ops->trap_group_set(devlink, group_item->group, policer,
7236                                            extack);
7237         if (err)
7238                 return err;
7239
7240         group_item->policer_item = policer_item;
7241
7242         return 0;
7243 }
7244
7245 static int devlink_nl_cmd_trap_group_set_doit(struct sk_buff *skb,
7246                                               struct genl_info *info)
7247 {
7248         struct netlink_ext_ack *extack = info->extack;
7249         struct devlink *devlink = info->user_ptr[0];
7250         struct devlink_trap_group_item *group_item;
7251         bool modified = false;
7252         int err;
7253
7254         if (list_empty(&devlink->trap_group_list))
7255                 return -EOPNOTSUPP;
7256
7257         group_item = devlink_trap_group_item_get_from_info(devlink, info);
7258         if (!group_item) {
7259                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
7260                 return -ENOENT;
7261         }
7262
7263         err = devlink_trap_group_action_set(devlink, group_item, info,
7264                                             &modified);
7265         if (err)
7266                 return err;
7267
7268         err = devlink_trap_group_set(devlink, group_item, info);
7269         if (err)
7270                 goto err_trap_group_set;
7271
7272         return 0;
7273
7274 err_trap_group_set:
7275         if (modified)
7276                 NL_SET_ERR_MSG_MOD(extack, "Trap group set failed, but some changes were committed already");
7277         return err;
7278 }
7279
7280 static struct devlink_trap_policer_item *
7281 devlink_trap_policer_item_get_from_info(struct devlink *devlink,
7282                                         struct genl_info *info)
7283 {
7284         u32 id;
7285
7286         if (!info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
7287                 return NULL;
7288         id = nla_get_u32(info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
7289
7290         return devlink_trap_policer_item_lookup(devlink, id);
7291 }
7292
7293 static int
7294 devlink_trap_policer_stats_put(struct sk_buff *msg, struct devlink *devlink,
7295                                const struct devlink_trap_policer *policer)
7296 {
7297         struct nlattr *attr;
7298         u64 drops;
7299         int err;
7300
7301         if (!devlink->ops->trap_policer_counter_get)
7302                 return 0;
7303
7304         err = devlink->ops->trap_policer_counter_get(devlink, policer, &drops);
7305         if (err)
7306                 return err;
7307
7308         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
7309         if (!attr)
7310                 return -EMSGSIZE;
7311
7312         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
7313                               DEVLINK_ATTR_PAD))
7314                 goto nla_put_failure;
7315
7316         nla_nest_end(msg, attr);
7317
7318         return 0;
7319
7320 nla_put_failure:
7321         nla_nest_cancel(msg, attr);
7322         return -EMSGSIZE;
7323 }
7324
7325 static int
7326 devlink_nl_trap_policer_fill(struct sk_buff *msg, struct devlink *devlink,
7327                              const struct devlink_trap_policer_item *policer_item,
7328                              enum devlink_command cmd, u32 portid, u32 seq,
7329                              int flags)
7330 {
7331         void *hdr;
7332         int err;
7333
7334         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
7335         if (!hdr)
7336                 return -EMSGSIZE;
7337
7338         if (devlink_nl_put_handle(msg, devlink))
7339                 goto nla_put_failure;
7340
7341         if (nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
7342                         policer_item->policer->id))
7343                 goto nla_put_failure;
7344
7345         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_RATE,
7346                               policer_item->rate, DEVLINK_ATTR_PAD))
7347                 goto nla_put_failure;
7348
7349         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_BURST,
7350                               policer_item->burst, DEVLINK_ATTR_PAD))
7351                 goto nla_put_failure;
7352
7353         err = devlink_trap_policer_stats_put(msg, devlink,
7354                                              policer_item->policer);
7355         if (err)
7356                 goto nla_put_failure;
7357
7358         genlmsg_end(msg, hdr);
7359
7360         return 0;
7361
7362 nla_put_failure:
7363         genlmsg_cancel(msg, hdr);
7364         return -EMSGSIZE;
7365 }
7366
7367 static int devlink_nl_cmd_trap_policer_get_doit(struct sk_buff *skb,
7368                                                 struct genl_info *info)
7369 {
7370         struct devlink_trap_policer_item *policer_item;
7371         struct netlink_ext_ack *extack = info->extack;
7372         struct devlink *devlink = info->user_ptr[0];
7373         struct sk_buff *msg;
7374         int err;
7375
7376         if (list_empty(&devlink->trap_policer_list))
7377                 return -EOPNOTSUPP;
7378
7379         policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
7380         if (!policer_item) {
7381                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
7382                 return -ENOENT;
7383         }
7384
7385         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7386         if (!msg)
7387                 return -ENOMEM;
7388
7389         err = devlink_nl_trap_policer_fill(msg, devlink, policer_item,
7390                                            DEVLINK_CMD_TRAP_POLICER_NEW,
7391                                            info->snd_portid, info->snd_seq, 0);
7392         if (err)
7393                 goto err_trap_policer_fill;
7394
7395         return genlmsg_reply(msg, info);
7396
7397 err_trap_policer_fill:
7398         nlmsg_free(msg);
7399         return err;
7400 }
7401
7402 static int devlink_nl_cmd_trap_policer_get_dumpit(struct sk_buff *msg,
7403                                                   struct netlink_callback *cb)
7404 {
7405         enum devlink_command cmd = DEVLINK_CMD_TRAP_POLICER_NEW;
7406         struct devlink_trap_policer_item *policer_item;
7407         u32 portid = NETLINK_CB(cb->skb).portid;
7408         struct devlink *devlink;
7409         int start = cb->args[0];
7410         int idx = 0;
7411         int err;
7412
7413         mutex_lock(&devlink_mutex);
7414         list_for_each_entry(devlink, &devlink_list, list) {
7415                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7416                         continue;
7417                 mutex_lock(&devlink->lock);
7418                 list_for_each_entry(policer_item, &devlink->trap_policer_list,
7419                                     list) {
7420                         if (idx < start) {
7421                                 idx++;
7422                                 continue;
7423                         }
7424                         err = devlink_nl_trap_policer_fill(msg, devlink,
7425                                                            policer_item, cmd,
7426                                                            portid,
7427                                                            cb->nlh->nlmsg_seq,
7428                                                            NLM_F_MULTI);
7429                         if (err) {
7430                                 mutex_unlock(&devlink->lock);
7431                                 goto out;
7432                         }
7433                         idx++;
7434                 }
7435                 mutex_unlock(&devlink->lock);
7436         }
7437 out:
7438         mutex_unlock(&devlink_mutex);
7439
7440         cb->args[0] = idx;
7441         return msg->len;
7442 }
7443
7444 static int
7445 devlink_trap_policer_set(struct devlink *devlink,
7446                          struct devlink_trap_policer_item *policer_item,
7447                          struct genl_info *info)
7448 {
7449         struct netlink_ext_ack *extack = info->extack;
7450         struct nlattr **attrs = info->attrs;
7451         u64 rate, burst;
7452         int err;
7453
7454         rate = policer_item->rate;
7455         burst = policer_item->burst;
7456
7457         if (attrs[DEVLINK_ATTR_TRAP_POLICER_RATE])
7458                 rate = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_RATE]);
7459
7460         if (attrs[DEVLINK_ATTR_TRAP_POLICER_BURST])
7461                 burst = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_BURST]);
7462
7463         if (rate < policer_item->policer->min_rate) {
7464                 NL_SET_ERR_MSG_MOD(extack, "Policer rate lower than limit");
7465                 return -EINVAL;
7466         }
7467
7468         if (rate > policer_item->policer->max_rate) {
7469                 NL_SET_ERR_MSG_MOD(extack, "Policer rate higher than limit");
7470                 return -EINVAL;
7471         }
7472
7473         if (burst < policer_item->policer->min_burst) {
7474                 NL_SET_ERR_MSG_MOD(extack, "Policer burst size lower than limit");
7475                 return -EINVAL;
7476         }
7477
7478         if (burst > policer_item->policer->max_burst) {
7479                 NL_SET_ERR_MSG_MOD(extack, "Policer burst size higher than limit");
7480                 return -EINVAL;
7481         }
7482
7483         err = devlink->ops->trap_policer_set(devlink, policer_item->policer,
7484                                              rate, burst, info->extack);
7485         if (err)
7486                 return err;
7487
7488         policer_item->rate = rate;
7489         policer_item->burst = burst;
7490
7491         return 0;
7492 }
7493
7494 static int devlink_nl_cmd_trap_policer_set_doit(struct sk_buff *skb,
7495                                                 struct genl_info *info)
7496 {
7497         struct devlink_trap_policer_item *policer_item;
7498         struct netlink_ext_ack *extack = info->extack;
7499         struct devlink *devlink = info->user_ptr[0];
7500
7501         if (list_empty(&devlink->trap_policer_list))
7502                 return -EOPNOTSUPP;
7503
7504         if (!devlink->ops->trap_policer_set)
7505                 return -EOPNOTSUPP;
7506
7507         policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
7508         if (!policer_item) {
7509                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
7510                 return -ENOENT;
7511         }
7512
7513         return devlink_trap_policer_set(devlink, policer_item, info);
7514 }
7515
7516 static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
7517         [DEVLINK_ATTR_UNSPEC] = { .strict_start_type =
7518                 DEVLINK_ATTR_TRAP_POLICER_ID },
7519         [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
7520         [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
7521         [DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32 },
7522         [DEVLINK_ATTR_PORT_TYPE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_PORT_TYPE_AUTO,
7523                                                     DEVLINK_PORT_TYPE_IB),
7524         [DEVLINK_ATTR_PORT_SPLIT_COUNT] = { .type = NLA_U32 },
7525         [DEVLINK_ATTR_SB_INDEX] = { .type = NLA_U32 },
7526         [DEVLINK_ATTR_SB_POOL_INDEX] = { .type = NLA_U16 },
7527         [DEVLINK_ATTR_SB_POOL_TYPE] = { .type = NLA_U8 },
7528         [DEVLINK_ATTR_SB_POOL_SIZE] = { .type = NLA_U32 },
7529         [DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE] = { .type = NLA_U8 },
7530         [DEVLINK_ATTR_SB_THRESHOLD] = { .type = NLA_U32 },
7531         [DEVLINK_ATTR_SB_TC_INDEX] = { .type = NLA_U16 },
7532         [DEVLINK_ATTR_ESWITCH_MODE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_ESWITCH_MODE_LEGACY,
7533                                                        DEVLINK_ESWITCH_MODE_SWITCHDEV),
7534         [DEVLINK_ATTR_ESWITCH_INLINE_MODE] = { .type = NLA_U8 },
7535         [DEVLINK_ATTR_ESWITCH_ENCAP_MODE] = { .type = NLA_U8 },
7536         [DEVLINK_ATTR_DPIPE_TABLE_NAME] = { .type = NLA_NUL_STRING },
7537         [DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED] = { .type = NLA_U8 },
7538         [DEVLINK_ATTR_RESOURCE_ID] = { .type = NLA_U64},
7539         [DEVLINK_ATTR_RESOURCE_SIZE] = { .type = NLA_U64},
7540         [DEVLINK_ATTR_PARAM_NAME] = { .type = NLA_NUL_STRING },
7541         [DEVLINK_ATTR_PARAM_TYPE] = { .type = NLA_U8 },
7542         [DEVLINK_ATTR_PARAM_VALUE_CMODE] = { .type = NLA_U8 },
7543         [DEVLINK_ATTR_REGION_NAME] = { .type = NLA_NUL_STRING },
7544         [DEVLINK_ATTR_REGION_SNAPSHOT_ID] = { .type = NLA_U32 },
7545         [DEVLINK_ATTR_REGION_CHUNK_ADDR] = { .type = NLA_U64 },
7546         [DEVLINK_ATTR_REGION_CHUNK_LEN] = { .type = NLA_U64 },
7547         [DEVLINK_ATTR_HEALTH_REPORTER_NAME] = { .type = NLA_NUL_STRING },
7548         [DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] = { .type = NLA_U64 },
7549         [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER] = { .type = NLA_U8 },
7550         [DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME] = { .type = NLA_NUL_STRING },
7551         [DEVLINK_ATTR_FLASH_UPDATE_COMPONENT] = { .type = NLA_NUL_STRING },
7552         [DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK] =
7553                 NLA_POLICY_BITFIELD32(DEVLINK_SUPPORTED_FLASH_OVERWRITE_SECTIONS),
7554         [DEVLINK_ATTR_TRAP_NAME] = { .type = NLA_NUL_STRING },
7555         [DEVLINK_ATTR_TRAP_ACTION] = { .type = NLA_U8 },
7556         [DEVLINK_ATTR_TRAP_GROUP_NAME] = { .type = NLA_NUL_STRING },
7557         [DEVLINK_ATTR_NETNS_PID] = { .type = NLA_U32 },
7558         [DEVLINK_ATTR_NETNS_FD] = { .type = NLA_U32 },
7559         [DEVLINK_ATTR_NETNS_ID] = { .type = NLA_U32 },
7560         [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP] = { .type = NLA_U8 },
7561         [DEVLINK_ATTR_TRAP_POLICER_ID] = { .type = NLA_U32 },
7562         [DEVLINK_ATTR_TRAP_POLICER_RATE] = { .type = NLA_U64 },
7563         [DEVLINK_ATTR_TRAP_POLICER_BURST] = { .type = NLA_U64 },
7564         [DEVLINK_ATTR_PORT_FUNCTION] = { .type = NLA_NESTED },
7565         [DEVLINK_ATTR_RELOAD_ACTION] = NLA_POLICY_RANGE(NLA_U8, DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
7566                                                         DEVLINK_RELOAD_ACTION_MAX),
7567         [DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(DEVLINK_RELOAD_LIMITS_VALID_MASK),
7568 };
7569
7570 static const struct genl_small_ops devlink_nl_ops[] = {
7571         {
7572                 .cmd = DEVLINK_CMD_GET,
7573                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7574                 .doit = devlink_nl_cmd_get_doit,
7575                 .dumpit = devlink_nl_cmd_get_dumpit,
7576                 /* can be retrieved by unprivileged users */
7577         },
7578         {
7579                 .cmd = DEVLINK_CMD_PORT_GET,
7580                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7581                 .doit = devlink_nl_cmd_port_get_doit,
7582                 .dumpit = devlink_nl_cmd_port_get_dumpit,
7583                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
7584                 /* can be retrieved by unprivileged users */
7585         },
7586         {
7587                 .cmd = DEVLINK_CMD_PORT_SET,
7588                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7589                 .doit = devlink_nl_cmd_port_set_doit,
7590                 .flags = GENL_ADMIN_PERM,
7591                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
7592         },
7593         {
7594                 .cmd = DEVLINK_CMD_PORT_SPLIT,
7595                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7596                 .doit = devlink_nl_cmd_port_split_doit,
7597                 .flags = GENL_ADMIN_PERM,
7598                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
7599         },
7600         {
7601                 .cmd = DEVLINK_CMD_PORT_UNSPLIT,
7602                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7603                 .doit = devlink_nl_cmd_port_unsplit_doit,
7604                 .flags = GENL_ADMIN_PERM,
7605                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
7606         },
7607         {
7608                 .cmd = DEVLINK_CMD_SB_GET,
7609                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7610                 .doit = devlink_nl_cmd_sb_get_doit,
7611                 .dumpit = devlink_nl_cmd_sb_get_dumpit,
7612                 /* can be retrieved by unprivileged users */
7613         },
7614         {
7615                 .cmd = DEVLINK_CMD_SB_POOL_GET,
7616                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7617                 .doit = devlink_nl_cmd_sb_pool_get_doit,
7618                 .dumpit = devlink_nl_cmd_sb_pool_get_dumpit,
7619                 /* can be retrieved by unprivileged users */
7620         },
7621         {
7622                 .cmd = DEVLINK_CMD_SB_POOL_SET,
7623                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7624                 .doit = devlink_nl_cmd_sb_pool_set_doit,
7625                 .flags = GENL_ADMIN_PERM,
7626         },
7627         {
7628                 .cmd = DEVLINK_CMD_SB_PORT_POOL_GET,
7629                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7630                 .doit = devlink_nl_cmd_sb_port_pool_get_doit,
7631                 .dumpit = devlink_nl_cmd_sb_port_pool_get_dumpit,
7632                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
7633                 /* can be retrieved by unprivileged users */
7634         },
7635         {
7636                 .cmd = DEVLINK_CMD_SB_PORT_POOL_SET,
7637                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7638                 .doit = devlink_nl_cmd_sb_port_pool_set_doit,
7639                 .flags = GENL_ADMIN_PERM,
7640                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
7641         },
7642         {
7643                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_GET,
7644                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7645                 .doit = devlink_nl_cmd_sb_tc_pool_bind_get_doit,
7646                 .dumpit = devlink_nl_cmd_sb_tc_pool_bind_get_dumpit,
7647                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
7648                 /* can be retrieved by unprivileged users */
7649         },
7650         {
7651                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_SET,
7652                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7653                 .doit = devlink_nl_cmd_sb_tc_pool_bind_set_doit,
7654                 .flags = GENL_ADMIN_PERM,
7655                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
7656         },
7657         {
7658                 .cmd = DEVLINK_CMD_SB_OCC_SNAPSHOT,
7659                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7660                 .doit = devlink_nl_cmd_sb_occ_snapshot_doit,
7661                 .flags = GENL_ADMIN_PERM,
7662         },
7663         {
7664                 .cmd = DEVLINK_CMD_SB_OCC_MAX_CLEAR,
7665                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7666                 .doit = devlink_nl_cmd_sb_occ_max_clear_doit,
7667                 .flags = GENL_ADMIN_PERM,
7668         },
7669         {
7670                 .cmd = DEVLINK_CMD_ESWITCH_GET,
7671                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7672                 .doit = devlink_nl_cmd_eswitch_get_doit,
7673                 .flags = GENL_ADMIN_PERM,
7674                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
7675         },
7676         {
7677                 .cmd = DEVLINK_CMD_ESWITCH_SET,
7678                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7679                 .doit = devlink_nl_cmd_eswitch_set_doit,
7680                 .flags = GENL_ADMIN_PERM,
7681                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
7682         },
7683         {
7684                 .cmd = DEVLINK_CMD_DPIPE_TABLE_GET,
7685                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7686                 .doit = devlink_nl_cmd_dpipe_table_get,
7687                 /* can be retrieved by unprivileged users */
7688         },
7689         {
7690                 .cmd = DEVLINK_CMD_DPIPE_ENTRIES_GET,
7691                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7692                 .doit = devlink_nl_cmd_dpipe_entries_get,
7693                 /* can be retrieved by unprivileged users */
7694         },
7695         {
7696                 .cmd = DEVLINK_CMD_DPIPE_HEADERS_GET,
7697                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7698                 .doit = devlink_nl_cmd_dpipe_headers_get,
7699                 /* can be retrieved by unprivileged users */
7700         },
7701         {
7702                 .cmd = DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET,
7703                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7704                 .doit = devlink_nl_cmd_dpipe_table_counters_set,
7705                 .flags = GENL_ADMIN_PERM,
7706         },
7707         {
7708                 .cmd = DEVLINK_CMD_RESOURCE_SET,
7709                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7710                 .doit = devlink_nl_cmd_resource_set,
7711                 .flags = GENL_ADMIN_PERM,
7712         },
7713         {
7714                 .cmd = DEVLINK_CMD_RESOURCE_DUMP,
7715                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7716                 .doit = devlink_nl_cmd_resource_dump,
7717                 /* can be retrieved by unprivileged users */
7718         },
7719         {
7720                 .cmd = DEVLINK_CMD_RELOAD,
7721                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7722                 .doit = devlink_nl_cmd_reload,
7723                 .flags = GENL_ADMIN_PERM,
7724                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
7725         },
7726         {
7727                 .cmd = DEVLINK_CMD_PARAM_GET,
7728                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7729                 .doit = devlink_nl_cmd_param_get_doit,
7730                 .dumpit = devlink_nl_cmd_param_get_dumpit,
7731                 /* can be retrieved by unprivileged users */
7732         },
7733         {
7734                 .cmd = DEVLINK_CMD_PARAM_SET,
7735                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7736                 .doit = devlink_nl_cmd_param_set_doit,
7737                 .flags = GENL_ADMIN_PERM,
7738         },
7739         {
7740                 .cmd = DEVLINK_CMD_PORT_PARAM_GET,
7741                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7742                 .doit = devlink_nl_cmd_port_param_get_doit,
7743                 .dumpit = devlink_nl_cmd_port_param_get_dumpit,
7744                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
7745                 /* can be retrieved by unprivileged users */
7746         },
7747         {
7748                 .cmd = DEVLINK_CMD_PORT_PARAM_SET,
7749                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7750                 .doit = devlink_nl_cmd_port_param_set_doit,
7751                 .flags = GENL_ADMIN_PERM,
7752                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
7753         },
7754         {
7755                 .cmd = DEVLINK_CMD_REGION_GET,
7756                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7757                 .doit = devlink_nl_cmd_region_get_doit,
7758                 .dumpit = devlink_nl_cmd_region_get_dumpit,
7759                 .flags = GENL_ADMIN_PERM,
7760         },
7761         {
7762                 .cmd = DEVLINK_CMD_REGION_NEW,
7763                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7764                 .doit = devlink_nl_cmd_region_new,
7765                 .flags = GENL_ADMIN_PERM,
7766         },
7767         {
7768                 .cmd = DEVLINK_CMD_REGION_DEL,
7769                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7770                 .doit = devlink_nl_cmd_region_del,
7771                 .flags = GENL_ADMIN_PERM,
7772         },
7773         {
7774                 .cmd = DEVLINK_CMD_REGION_READ,
7775                 .validate = GENL_DONT_VALIDATE_STRICT |
7776                             GENL_DONT_VALIDATE_DUMP_STRICT,
7777                 .dumpit = devlink_nl_cmd_region_read_dumpit,
7778                 .flags = GENL_ADMIN_PERM,
7779         },
7780         {
7781                 .cmd = DEVLINK_CMD_INFO_GET,
7782                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7783                 .doit = devlink_nl_cmd_info_get_doit,
7784                 .dumpit = devlink_nl_cmd_info_get_dumpit,
7785                 /* can be retrieved by unprivileged users */
7786         },
7787         {
7788                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_GET,
7789                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7790                 .doit = devlink_nl_cmd_health_reporter_get_doit,
7791                 .dumpit = devlink_nl_cmd_health_reporter_get_dumpit,
7792                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
7793                                   DEVLINK_NL_FLAG_NO_LOCK,
7794                 /* can be retrieved by unprivileged users */
7795         },
7796         {
7797                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_SET,
7798                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7799                 .doit = devlink_nl_cmd_health_reporter_set_doit,
7800                 .flags = GENL_ADMIN_PERM,
7801                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
7802                                   DEVLINK_NL_FLAG_NO_LOCK,
7803         },
7804         {
7805                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_RECOVER,
7806                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7807                 .doit = devlink_nl_cmd_health_reporter_recover_doit,
7808                 .flags = GENL_ADMIN_PERM,
7809                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
7810                                   DEVLINK_NL_FLAG_NO_LOCK,
7811         },
7812         {
7813                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE,
7814                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7815                 .doit = devlink_nl_cmd_health_reporter_diagnose_doit,
7816                 .flags = GENL_ADMIN_PERM,
7817                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
7818                                   DEVLINK_NL_FLAG_NO_LOCK,
7819         },
7820         {
7821                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET,
7822                 .validate = GENL_DONT_VALIDATE_STRICT |
7823                             GENL_DONT_VALIDATE_DUMP_STRICT,
7824                 .dumpit = devlink_nl_cmd_health_reporter_dump_get_dumpit,
7825                 .flags = GENL_ADMIN_PERM,
7826                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
7827                                   DEVLINK_NL_FLAG_NO_LOCK,
7828         },
7829         {
7830                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR,
7831                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7832                 .doit = devlink_nl_cmd_health_reporter_dump_clear_doit,
7833                 .flags = GENL_ADMIN_PERM,
7834                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
7835                                   DEVLINK_NL_FLAG_NO_LOCK,
7836         },
7837         {
7838                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_TEST,
7839                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7840                 .doit = devlink_nl_cmd_health_reporter_test_doit,
7841                 .flags = GENL_ADMIN_PERM,
7842                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
7843                                   DEVLINK_NL_FLAG_NO_LOCK,
7844         },
7845         {
7846                 .cmd = DEVLINK_CMD_FLASH_UPDATE,
7847                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
7848                 .doit = devlink_nl_cmd_flash_update,
7849                 .flags = GENL_ADMIN_PERM,
7850         },
7851         {
7852                 .cmd = DEVLINK_CMD_TRAP_GET,
7853                 .doit = devlink_nl_cmd_trap_get_doit,
7854                 .dumpit = devlink_nl_cmd_trap_get_dumpit,
7855                 /* can be retrieved by unprivileged users */
7856         },
7857         {
7858                 .cmd = DEVLINK_CMD_TRAP_SET,
7859                 .doit = devlink_nl_cmd_trap_set_doit,
7860                 .flags = GENL_ADMIN_PERM,
7861         },
7862         {
7863                 .cmd = DEVLINK_CMD_TRAP_GROUP_GET,
7864                 .doit = devlink_nl_cmd_trap_group_get_doit,
7865                 .dumpit = devlink_nl_cmd_trap_group_get_dumpit,
7866                 /* can be retrieved by unprivileged users */
7867         },
7868         {
7869                 .cmd = DEVLINK_CMD_TRAP_GROUP_SET,
7870                 .doit = devlink_nl_cmd_trap_group_set_doit,
7871                 .flags = GENL_ADMIN_PERM,
7872         },
7873         {
7874                 .cmd = DEVLINK_CMD_TRAP_POLICER_GET,
7875                 .doit = devlink_nl_cmd_trap_policer_get_doit,
7876                 .dumpit = devlink_nl_cmd_trap_policer_get_dumpit,
7877                 /* can be retrieved by unprivileged users */
7878         },
7879         {
7880                 .cmd = DEVLINK_CMD_TRAP_POLICER_SET,
7881                 .doit = devlink_nl_cmd_trap_policer_set_doit,
7882                 .flags = GENL_ADMIN_PERM,
7883         },
7884 };
7885
7886 static struct genl_family devlink_nl_family __ro_after_init = {
7887         .name           = DEVLINK_GENL_NAME,
7888         .version        = DEVLINK_GENL_VERSION,
7889         .maxattr        = DEVLINK_ATTR_MAX,
7890         .policy = devlink_nl_policy,
7891         .netnsok        = true,
7892         .pre_doit       = devlink_nl_pre_doit,
7893         .post_doit      = devlink_nl_post_doit,
7894         .module         = THIS_MODULE,
7895         .small_ops      = devlink_nl_ops,
7896         .n_small_ops    = ARRAY_SIZE(devlink_nl_ops),
7897         .mcgrps         = devlink_nl_mcgrps,
7898         .n_mcgrps       = ARRAY_SIZE(devlink_nl_mcgrps),
7899 };
7900
7901 static bool devlink_reload_actions_valid(const struct devlink_ops *ops)
7902 {
7903         const struct devlink_reload_combination *comb;
7904         int i;
7905
7906         if (!devlink_reload_supported(ops)) {
7907                 if (WARN_ON(ops->reload_actions))
7908                         return false;
7909                 return true;
7910         }
7911
7912         if (WARN_ON(!ops->reload_actions ||
7913                     ops->reload_actions & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
7914                     ops->reload_actions >= BIT(__DEVLINK_RELOAD_ACTION_MAX)))
7915                 return false;
7916
7917         if (WARN_ON(ops->reload_limits & BIT(DEVLINK_RELOAD_LIMIT_UNSPEC) ||
7918                     ops->reload_limits >= BIT(__DEVLINK_RELOAD_LIMIT_MAX)))
7919                 return false;
7920
7921         for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)  {
7922                 comb = &devlink_reload_invalid_combinations[i];
7923                 if (ops->reload_actions == BIT(comb->action) &&
7924                     ops->reload_limits == BIT(comb->limit))
7925                         return false;
7926         }
7927         return true;
7928 }
7929
7930 /**
7931  *      devlink_alloc - Allocate new devlink instance resources
7932  *
7933  *      @ops: ops
7934  *      @priv_size: size of user private data
7935  *
7936  *      Allocate new devlink instance resources, including devlink index
7937  *      and name.
7938  */
7939 struct devlink *devlink_alloc(const struct devlink_ops *ops, size_t priv_size)
7940 {
7941         struct devlink *devlink;
7942
7943         if (WARN_ON(!ops))
7944                 return NULL;
7945
7946         if (!devlink_reload_actions_valid(ops))
7947                 return NULL;
7948
7949         devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL);
7950         if (!devlink)
7951                 return NULL;
7952         devlink->ops = ops;
7953         xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
7954         __devlink_net_set(devlink, &init_net);
7955         INIT_LIST_HEAD(&devlink->port_list);
7956         INIT_LIST_HEAD(&devlink->sb_list);
7957         INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
7958         INIT_LIST_HEAD(&devlink->resource_list);
7959         INIT_LIST_HEAD(&devlink->param_list);
7960         INIT_LIST_HEAD(&devlink->region_list);
7961         INIT_LIST_HEAD(&devlink->reporter_list);
7962         INIT_LIST_HEAD(&devlink->trap_list);
7963         INIT_LIST_HEAD(&devlink->trap_group_list);
7964         INIT_LIST_HEAD(&devlink->trap_policer_list);
7965         mutex_init(&devlink->lock);
7966         mutex_init(&devlink->reporters_lock);
7967         return devlink;
7968 }
7969 EXPORT_SYMBOL_GPL(devlink_alloc);
7970
7971 /**
7972  *      devlink_register - Register devlink instance
7973  *
7974  *      @devlink: devlink
7975  *      @dev: parent device
7976  */
7977 int devlink_register(struct devlink *devlink, struct device *dev)
7978 {
7979         devlink->dev = dev;
7980         devlink->registered = true;
7981         mutex_lock(&devlink_mutex);
7982         list_add_tail(&devlink->list, &devlink_list);
7983         devlink_notify(devlink, DEVLINK_CMD_NEW);
7984         mutex_unlock(&devlink_mutex);
7985         return 0;
7986 }
7987 EXPORT_SYMBOL_GPL(devlink_register);
7988
7989 /**
7990  *      devlink_unregister - Unregister devlink instance
7991  *
7992  *      @devlink: devlink
7993  */
7994 void devlink_unregister(struct devlink *devlink)
7995 {
7996         mutex_lock(&devlink_mutex);
7997         WARN_ON(devlink_reload_supported(devlink->ops) &&
7998                 devlink->reload_enabled);
7999         devlink_notify(devlink, DEVLINK_CMD_DEL);
8000         list_del(&devlink->list);
8001         mutex_unlock(&devlink_mutex);
8002 }
8003 EXPORT_SYMBOL_GPL(devlink_unregister);
8004
8005 /**
8006  *      devlink_reload_enable - Enable reload of devlink instance
8007  *
8008  *      @devlink: devlink
8009  *
8010  *      Should be called at end of device initialization
8011  *      process when reload operation is supported.
8012  */
8013 void devlink_reload_enable(struct devlink *devlink)
8014 {
8015         mutex_lock(&devlink_mutex);
8016         devlink->reload_enabled = true;
8017         mutex_unlock(&devlink_mutex);
8018 }
8019 EXPORT_SYMBOL_GPL(devlink_reload_enable);
8020
8021 /**
8022  *      devlink_reload_disable - Disable reload of devlink instance
8023  *
8024  *      @devlink: devlink
8025  *
8026  *      Should be called at the beginning of device cleanup
8027  *      process when reload operation is supported.
8028  */
8029 void devlink_reload_disable(struct devlink *devlink)
8030 {
8031         mutex_lock(&devlink_mutex);
8032         /* Mutex is taken which ensures that no reload operation is in
8033          * progress while setting up forbidded flag.
8034          */
8035         devlink->reload_enabled = false;
8036         mutex_unlock(&devlink_mutex);
8037 }
8038 EXPORT_SYMBOL_GPL(devlink_reload_disable);
8039
8040 /**
8041  *      devlink_free - Free devlink instance resources
8042  *
8043  *      @devlink: devlink
8044  */
8045 void devlink_free(struct devlink *devlink)
8046 {
8047         mutex_destroy(&devlink->reporters_lock);
8048         mutex_destroy(&devlink->lock);
8049         WARN_ON(!list_empty(&devlink->trap_policer_list));
8050         WARN_ON(!list_empty(&devlink->trap_group_list));
8051         WARN_ON(!list_empty(&devlink->trap_list));
8052         WARN_ON(!list_empty(&devlink->reporter_list));
8053         WARN_ON(!list_empty(&devlink->region_list));
8054         WARN_ON(!list_empty(&devlink->param_list));
8055         WARN_ON(!list_empty(&devlink->resource_list));
8056         WARN_ON(!list_empty(&devlink->dpipe_table_list));
8057         WARN_ON(!list_empty(&devlink->sb_list));
8058         WARN_ON(!list_empty(&devlink->port_list));
8059
8060         xa_destroy(&devlink->snapshot_ids);
8061
8062         kfree(devlink);
8063 }
8064 EXPORT_SYMBOL_GPL(devlink_free);
8065
8066 static void devlink_port_type_warn(struct work_struct *work)
8067 {
8068         WARN(true, "Type was not set for devlink port.");
8069 }
8070
8071 static bool devlink_port_type_should_warn(struct devlink_port *devlink_port)
8072 {
8073         /* Ignore CPU and DSA flavours. */
8074         return devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_CPU &&
8075                devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_DSA &&
8076                devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_UNUSED;
8077 }
8078
8079 #define DEVLINK_PORT_TYPE_WARN_TIMEOUT (HZ * 3600)
8080
8081 static void devlink_port_type_warn_schedule(struct devlink_port *devlink_port)
8082 {
8083         if (!devlink_port_type_should_warn(devlink_port))
8084                 return;
8085         /* Schedule a work to WARN in case driver does not set port
8086          * type within timeout.
8087          */
8088         schedule_delayed_work(&devlink_port->type_warn_dw,
8089                               DEVLINK_PORT_TYPE_WARN_TIMEOUT);
8090 }
8091
8092 static void devlink_port_type_warn_cancel(struct devlink_port *devlink_port)
8093 {
8094         if (!devlink_port_type_should_warn(devlink_port))
8095                 return;
8096         cancel_delayed_work_sync(&devlink_port->type_warn_dw);
8097 }
8098
8099 /**
8100  *      devlink_port_register - Register devlink port
8101  *
8102  *      @devlink: devlink
8103  *      @devlink_port: devlink port
8104  *      @port_index: driver-specific numerical identifier of the port
8105  *
8106  *      Register devlink port with provided port index. User can use
8107  *      any indexing, even hw-related one. devlink_port structure
8108  *      is convenient to be embedded inside user driver private structure.
8109  *      Note that the caller should take care of zeroing the devlink_port
8110  *      structure.
8111  */
8112 int devlink_port_register(struct devlink *devlink,
8113                           struct devlink_port *devlink_port,
8114                           unsigned int port_index)
8115 {
8116         mutex_lock(&devlink->lock);
8117         if (devlink_port_index_exists(devlink, port_index)) {
8118                 mutex_unlock(&devlink->lock);
8119                 return -EEXIST;
8120         }
8121         devlink_port->devlink = devlink;
8122         devlink_port->index = port_index;
8123         devlink_port->registered = true;
8124         spin_lock_init(&devlink_port->type_lock);
8125         INIT_LIST_HEAD(&devlink_port->reporter_list);
8126         mutex_init(&devlink_port->reporters_lock);
8127         list_add_tail(&devlink_port->list, &devlink->port_list);
8128         INIT_LIST_HEAD(&devlink_port->param_list);
8129         INIT_LIST_HEAD(&devlink_port->region_list);
8130         mutex_unlock(&devlink->lock);
8131         INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn);
8132         devlink_port_type_warn_schedule(devlink_port);
8133         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
8134         return 0;
8135 }
8136 EXPORT_SYMBOL_GPL(devlink_port_register);
8137
8138 /**
8139  *      devlink_port_unregister - Unregister devlink port
8140  *
8141  *      @devlink_port: devlink port
8142  */
8143 void devlink_port_unregister(struct devlink_port *devlink_port)
8144 {
8145         struct devlink *devlink = devlink_port->devlink;
8146
8147         devlink_port_type_warn_cancel(devlink_port);
8148         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
8149         mutex_lock(&devlink->lock);
8150         list_del(&devlink_port->list);
8151         mutex_unlock(&devlink->lock);
8152         WARN_ON(!list_empty(&devlink_port->reporter_list));
8153         WARN_ON(!list_empty(&devlink_port->region_list));
8154         mutex_destroy(&devlink_port->reporters_lock);
8155 }
8156 EXPORT_SYMBOL_GPL(devlink_port_unregister);
8157
8158 static void __devlink_port_type_set(struct devlink_port *devlink_port,
8159                                     enum devlink_port_type type,
8160                                     void *type_dev)
8161 {
8162         if (WARN_ON(!devlink_port->registered))
8163                 return;
8164         devlink_port_type_warn_cancel(devlink_port);
8165         spin_lock_bh(&devlink_port->type_lock);
8166         devlink_port->type = type;
8167         devlink_port->type_dev = type_dev;
8168         spin_unlock_bh(&devlink_port->type_lock);
8169         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
8170 }
8171
8172 static void devlink_port_type_netdev_checks(struct devlink_port *devlink_port,
8173                                             struct net_device *netdev)
8174 {
8175         const struct net_device_ops *ops = netdev->netdev_ops;
8176
8177         /* If driver registers devlink port, it should set devlink port
8178          * attributes accordingly so the compat functions are called
8179          * and the original ops are not used.
8180          */
8181         if (ops->ndo_get_phys_port_name) {
8182                 /* Some drivers use the same set of ndos for netdevs
8183                  * that have devlink_port registered and also for
8184                  * those who don't. Make sure that ndo_get_phys_port_name
8185                  * returns -EOPNOTSUPP here in case it is defined.
8186                  * Warn if not.
8187                  */
8188                 char name[IFNAMSIZ];
8189                 int err;
8190
8191                 err = ops->ndo_get_phys_port_name(netdev, name, sizeof(name));
8192                 WARN_ON(err != -EOPNOTSUPP);
8193         }
8194         if (ops->ndo_get_port_parent_id) {
8195                 /* Some drivers use the same set of ndos for netdevs
8196                  * that have devlink_port registered and also for
8197                  * those who don't. Make sure that ndo_get_port_parent_id
8198                  * returns -EOPNOTSUPP here in case it is defined.
8199                  * Warn if not.
8200                  */
8201                 struct netdev_phys_item_id ppid;
8202                 int err;
8203
8204                 err = ops->ndo_get_port_parent_id(netdev, &ppid);
8205                 WARN_ON(err != -EOPNOTSUPP);
8206         }
8207 }
8208
8209 /**
8210  *      devlink_port_type_eth_set - Set port type to Ethernet
8211  *
8212  *      @devlink_port: devlink port
8213  *      @netdev: related netdevice
8214  */
8215 void devlink_port_type_eth_set(struct devlink_port *devlink_port,
8216                                struct net_device *netdev)
8217 {
8218         if (netdev)
8219                 devlink_port_type_netdev_checks(devlink_port, netdev);
8220         else
8221                 dev_warn(devlink_port->devlink->dev,
8222                          "devlink port type for port %d set to Ethernet without a software interface reference, device type not supported by the kernel?\n",
8223                          devlink_port->index);
8224
8225         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_ETH, netdev);
8226 }
8227 EXPORT_SYMBOL_GPL(devlink_port_type_eth_set);
8228
8229 /**
8230  *      devlink_port_type_ib_set - Set port type to InfiniBand
8231  *
8232  *      @devlink_port: devlink port
8233  *      @ibdev: related IB device
8234  */
8235 void devlink_port_type_ib_set(struct devlink_port *devlink_port,
8236                               struct ib_device *ibdev)
8237 {
8238         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_IB, ibdev);
8239 }
8240 EXPORT_SYMBOL_GPL(devlink_port_type_ib_set);
8241
8242 /**
8243  *      devlink_port_type_clear - Clear port type
8244  *
8245  *      @devlink_port: devlink port
8246  */
8247 void devlink_port_type_clear(struct devlink_port *devlink_port)
8248 {
8249         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_NOTSET, NULL);
8250         devlink_port_type_warn_schedule(devlink_port);
8251 }
8252 EXPORT_SYMBOL_GPL(devlink_port_type_clear);
8253
8254 static int __devlink_port_attrs_set(struct devlink_port *devlink_port,
8255                                     enum devlink_port_flavour flavour)
8256 {
8257         struct devlink_port_attrs *attrs = &devlink_port->attrs;
8258
8259         devlink_port->attrs_set = true;
8260         attrs->flavour = flavour;
8261         if (attrs->switch_id.id_len) {
8262                 devlink_port->switch_port = true;
8263                 if (WARN_ON(attrs->switch_id.id_len > MAX_PHYS_ITEM_ID_LEN))
8264                         attrs->switch_id.id_len = MAX_PHYS_ITEM_ID_LEN;
8265         } else {
8266                 devlink_port->switch_port = false;
8267         }
8268         return 0;
8269 }
8270
8271 /**
8272  *      devlink_port_attrs_set - Set port attributes
8273  *
8274  *      @devlink_port: devlink port
8275  *      @attrs: devlink port attrs
8276  */
8277 void devlink_port_attrs_set(struct devlink_port *devlink_port,
8278                             struct devlink_port_attrs *attrs)
8279 {
8280         int ret;
8281
8282         if (WARN_ON(devlink_port->registered))
8283                 return;
8284         devlink_port->attrs = *attrs;
8285         ret = __devlink_port_attrs_set(devlink_port, attrs->flavour);
8286         if (ret)
8287                 return;
8288         WARN_ON(attrs->splittable && attrs->split);
8289 }
8290 EXPORT_SYMBOL_GPL(devlink_port_attrs_set);
8291
8292 /**
8293  *      devlink_port_attrs_pci_pf_set - Set PCI PF port attributes
8294  *
8295  *      @devlink_port: devlink port
8296  *      @controller: associated controller number for the devlink port instance
8297  *      @pf: associated PF for the devlink port instance
8298  *      @external: indicates if the port is for an external controller
8299  */
8300 void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 controller,
8301                                    u16 pf, bool external)
8302 {
8303         struct devlink_port_attrs *attrs = &devlink_port->attrs;
8304         int ret;
8305
8306         if (WARN_ON(devlink_port->registered))
8307                 return;
8308         ret = __devlink_port_attrs_set(devlink_port,
8309                                        DEVLINK_PORT_FLAVOUR_PCI_PF);
8310         if (ret)
8311                 return;
8312         attrs->pci_pf.controller = controller;
8313         attrs->pci_pf.pf = pf;
8314         attrs->pci_pf.external = external;
8315 }
8316 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_pf_set);
8317
8318 /**
8319  *      devlink_port_attrs_pci_vf_set - Set PCI VF port attributes
8320  *
8321  *      @devlink_port: devlink port
8322  *      @controller: associated controller number for the devlink port instance
8323  *      @pf: associated PF for the devlink port instance
8324  *      @vf: associated VF of a PF for the devlink port instance
8325  *      @external: indicates if the port is for an external controller
8326  */
8327 void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller,
8328                                    u16 pf, u16 vf, bool external)
8329 {
8330         struct devlink_port_attrs *attrs = &devlink_port->attrs;
8331         int ret;
8332
8333         if (WARN_ON(devlink_port->registered))
8334                 return;
8335         ret = __devlink_port_attrs_set(devlink_port,
8336                                        DEVLINK_PORT_FLAVOUR_PCI_VF);
8337         if (ret)
8338                 return;
8339         attrs->pci_vf.controller = controller;
8340         attrs->pci_vf.pf = pf;
8341         attrs->pci_vf.vf = vf;
8342         attrs->pci_vf.external = external;
8343 }
8344 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_vf_set);
8345
8346 static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
8347                                              char *name, size_t len)
8348 {
8349         struct devlink_port_attrs *attrs = &devlink_port->attrs;
8350         int n = 0;
8351
8352         if (!devlink_port->attrs_set)
8353                 return -EOPNOTSUPP;
8354
8355         switch (attrs->flavour) {
8356         case DEVLINK_PORT_FLAVOUR_PHYSICAL:
8357         case DEVLINK_PORT_FLAVOUR_VIRTUAL:
8358                 if (!attrs->split)
8359                         n = snprintf(name, len, "p%u", attrs->phys.port_number);
8360                 else
8361                         n = snprintf(name, len, "p%us%u",
8362                                      attrs->phys.port_number,
8363                                      attrs->phys.split_subport_number);
8364                 break;
8365         case DEVLINK_PORT_FLAVOUR_CPU:
8366         case DEVLINK_PORT_FLAVOUR_DSA:
8367         case DEVLINK_PORT_FLAVOUR_UNUSED:
8368                 /* As CPU and DSA ports do not have a netdevice associated
8369                  * case should not ever happen.
8370                  */
8371                 WARN_ON(1);
8372                 return -EINVAL;
8373         case DEVLINK_PORT_FLAVOUR_PCI_PF:
8374                 if (attrs->pci_pf.external) {
8375                         n = snprintf(name, len, "c%u", attrs->pci_pf.controller);
8376                         if (n >= len)
8377                                 return -EINVAL;
8378                         len -= n;
8379                         name += n;
8380                 }
8381                 n = snprintf(name, len, "pf%u", attrs->pci_pf.pf);
8382                 break;
8383         case DEVLINK_PORT_FLAVOUR_PCI_VF:
8384                 if (attrs->pci_vf.external) {
8385                         n = snprintf(name, len, "c%u", attrs->pci_vf.controller);
8386                         if (n >= len)
8387                                 return -EINVAL;
8388                         len -= n;
8389                         name += n;
8390                 }
8391                 n = snprintf(name, len, "pf%uvf%u",
8392                              attrs->pci_vf.pf, attrs->pci_vf.vf);
8393                 break;
8394         }
8395
8396         if (n >= len)
8397                 return -EINVAL;
8398
8399         return 0;
8400 }
8401
8402 int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
8403                         u32 size, u16 ingress_pools_count,
8404                         u16 egress_pools_count, u16 ingress_tc_count,
8405                         u16 egress_tc_count)
8406 {
8407         struct devlink_sb *devlink_sb;
8408         int err = 0;
8409
8410         mutex_lock(&devlink->lock);
8411         if (devlink_sb_index_exists(devlink, sb_index)) {
8412                 err = -EEXIST;
8413                 goto unlock;
8414         }
8415
8416         devlink_sb = kzalloc(sizeof(*devlink_sb), GFP_KERNEL);
8417         if (!devlink_sb) {
8418                 err = -ENOMEM;
8419                 goto unlock;
8420         }
8421         devlink_sb->index = sb_index;
8422         devlink_sb->size = size;
8423         devlink_sb->ingress_pools_count = ingress_pools_count;
8424         devlink_sb->egress_pools_count = egress_pools_count;
8425         devlink_sb->ingress_tc_count = ingress_tc_count;
8426         devlink_sb->egress_tc_count = egress_tc_count;
8427         list_add_tail(&devlink_sb->list, &devlink->sb_list);
8428 unlock:
8429         mutex_unlock(&devlink->lock);
8430         return err;
8431 }
8432 EXPORT_SYMBOL_GPL(devlink_sb_register);
8433
8434 void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index)
8435 {
8436         struct devlink_sb *devlink_sb;
8437
8438         mutex_lock(&devlink->lock);
8439         devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
8440         WARN_ON(!devlink_sb);
8441         list_del(&devlink_sb->list);
8442         mutex_unlock(&devlink->lock);
8443         kfree(devlink_sb);
8444 }
8445 EXPORT_SYMBOL_GPL(devlink_sb_unregister);
8446
8447 /**
8448  *      devlink_dpipe_headers_register - register dpipe headers
8449  *
8450  *      @devlink: devlink
8451  *      @dpipe_headers: dpipe header array
8452  *
8453  *      Register the headers supported by hardware.
8454  */
8455 int devlink_dpipe_headers_register(struct devlink *devlink,
8456                                    struct devlink_dpipe_headers *dpipe_headers)
8457 {
8458         mutex_lock(&devlink->lock);
8459         devlink->dpipe_headers = dpipe_headers;
8460         mutex_unlock(&devlink->lock);
8461         return 0;
8462 }
8463 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_register);
8464
8465 /**
8466  *      devlink_dpipe_headers_unregister - unregister dpipe headers
8467  *
8468  *      @devlink: devlink
8469  *
8470  *      Unregister the headers supported by hardware.
8471  */
8472 void devlink_dpipe_headers_unregister(struct devlink *devlink)
8473 {
8474         mutex_lock(&devlink->lock);
8475         devlink->dpipe_headers = NULL;
8476         mutex_unlock(&devlink->lock);
8477 }
8478 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_unregister);
8479
8480 /**
8481  *      devlink_dpipe_table_counter_enabled - check if counter allocation
8482  *                                            required
8483  *      @devlink: devlink
8484  *      @table_name: tables name
8485  *
8486  *      Used by driver to check if counter allocation is required.
8487  *      After counter allocation is turned on the table entries
8488  *      are updated to include counter statistics.
8489  *
8490  *      After that point on the driver must respect the counter
8491  *      state so that each entry added to the table is added
8492  *      with a counter.
8493  */
8494 bool devlink_dpipe_table_counter_enabled(struct devlink *devlink,
8495                                          const char *table_name)
8496 {
8497         struct devlink_dpipe_table *table;
8498         bool enabled;
8499
8500         rcu_read_lock();
8501         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
8502                                          table_name, devlink);
8503         enabled = false;
8504         if (table)
8505                 enabled = table->counters_enabled;
8506         rcu_read_unlock();
8507         return enabled;
8508 }
8509 EXPORT_SYMBOL_GPL(devlink_dpipe_table_counter_enabled);
8510
8511 /**
8512  *      devlink_dpipe_table_register - register dpipe table
8513  *
8514  *      @devlink: devlink
8515  *      @table_name: table name
8516  *      @table_ops: table ops
8517  *      @priv: priv
8518  *      @counter_control_extern: external control for counters
8519  */
8520 int devlink_dpipe_table_register(struct devlink *devlink,
8521                                  const char *table_name,
8522                                  struct devlink_dpipe_table_ops *table_ops,
8523                                  void *priv, bool counter_control_extern)
8524 {
8525         struct devlink_dpipe_table *table;
8526         int err = 0;
8527
8528         if (WARN_ON(!table_ops->size_get))
8529                 return -EINVAL;
8530
8531         mutex_lock(&devlink->lock);
8532
8533         if (devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name,
8534                                      devlink)) {
8535                 err = -EEXIST;
8536                 goto unlock;
8537         }
8538
8539         table = kzalloc(sizeof(*table), GFP_KERNEL);
8540         if (!table) {
8541                 err = -ENOMEM;
8542                 goto unlock;
8543         }
8544
8545         table->name = table_name;
8546         table->table_ops = table_ops;
8547         table->priv = priv;
8548         table->counter_control_extern = counter_control_extern;
8549
8550         list_add_tail_rcu(&table->list, &devlink->dpipe_table_list);
8551 unlock:
8552         mutex_unlock(&devlink->lock);
8553         return err;
8554 }
8555 EXPORT_SYMBOL_GPL(devlink_dpipe_table_register);
8556
8557 /**
8558  *      devlink_dpipe_table_unregister - unregister dpipe table
8559  *
8560  *      @devlink: devlink
8561  *      @table_name: table name
8562  */
8563 void devlink_dpipe_table_unregister(struct devlink *devlink,
8564                                     const char *table_name)
8565 {
8566         struct devlink_dpipe_table *table;
8567
8568         mutex_lock(&devlink->lock);
8569         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
8570                                          table_name, devlink);
8571         if (!table)
8572                 goto unlock;
8573         list_del_rcu(&table->list);
8574         mutex_unlock(&devlink->lock);
8575         kfree_rcu(table, rcu);
8576         return;
8577 unlock:
8578         mutex_unlock(&devlink->lock);
8579 }
8580 EXPORT_SYMBOL_GPL(devlink_dpipe_table_unregister);
8581
8582 /**
8583  *      devlink_resource_register - devlink resource register
8584  *
8585  *      @devlink: devlink
8586  *      @resource_name: resource's name
8587  *      @resource_size: resource's size
8588  *      @resource_id: resource's id
8589  *      @parent_resource_id: resource's parent id
8590  *      @size_params: size parameters
8591  */
8592 int devlink_resource_register(struct devlink *devlink,
8593                               const char *resource_name,
8594                               u64 resource_size,
8595                               u64 resource_id,
8596                               u64 parent_resource_id,
8597                               const struct devlink_resource_size_params *size_params)
8598 {
8599         struct devlink_resource *resource;
8600         struct list_head *resource_list;
8601         bool top_hierarchy;
8602         int err = 0;
8603
8604         top_hierarchy = parent_resource_id == DEVLINK_RESOURCE_ID_PARENT_TOP;
8605
8606         mutex_lock(&devlink->lock);
8607         resource = devlink_resource_find(devlink, NULL, resource_id);
8608         if (resource) {
8609                 err = -EINVAL;
8610                 goto out;
8611         }
8612
8613         resource = kzalloc(sizeof(*resource), GFP_KERNEL);
8614         if (!resource) {
8615                 err = -ENOMEM;
8616                 goto out;
8617         }
8618
8619         if (top_hierarchy) {
8620                 resource_list = &devlink->resource_list;
8621         } else {
8622                 struct devlink_resource *parent_resource;
8623
8624                 parent_resource = devlink_resource_find(devlink, NULL,
8625                                                         parent_resource_id);
8626                 if (parent_resource) {
8627                         resource_list = &parent_resource->resource_list;
8628                         resource->parent = parent_resource;
8629                 } else {
8630                         kfree(resource);
8631                         err = -EINVAL;
8632                         goto out;
8633                 }
8634         }
8635
8636         resource->name = resource_name;
8637         resource->size = resource_size;
8638         resource->size_new = resource_size;
8639         resource->id = resource_id;
8640         resource->size_valid = true;
8641         memcpy(&resource->size_params, size_params,
8642                sizeof(resource->size_params));
8643         INIT_LIST_HEAD(&resource->resource_list);
8644         list_add_tail(&resource->list, resource_list);
8645 out:
8646         mutex_unlock(&devlink->lock);
8647         return err;
8648 }
8649 EXPORT_SYMBOL_GPL(devlink_resource_register);
8650
8651 /**
8652  *      devlink_resources_unregister - free all resources
8653  *
8654  *      @devlink: devlink
8655  *      @resource: resource
8656  */
8657 void devlink_resources_unregister(struct devlink *devlink,
8658                                   struct devlink_resource *resource)
8659 {
8660         struct devlink_resource *tmp, *child_resource;
8661         struct list_head *resource_list;
8662
8663         if (resource)
8664                 resource_list = &resource->resource_list;
8665         else
8666                 resource_list = &devlink->resource_list;
8667
8668         if (!resource)
8669                 mutex_lock(&devlink->lock);
8670
8671         list_for_each_entry_safe(child_resource, tmp, resource_list, list) {
8672                 devlink_resources_unregister(devlink, child_resource);
8673                 list_del(&child_resource->list);
8674                 kfree(child_resource);
8675         }
8676
8677         if (!resource)
8678                 mutex_unlock(&devlink->lock);
8679 }
8680 EXPORT_SYMBOL_GPL(devlink_resources_unregister);
8681
8682 /**
8683  *      devlink_resource_size_get - get and update size
8684  *
8685  *      @devlink: devlink
8686  *      @resource_id: the requested resource id
8687  *      @p_resource_size: ptr to update
8688  */
8689 int devlink_resource_size_get(struct devlink *devlink,
8690                               u64 resource_id,
8691                               u64 *p_resource_size)
8692 {
8693         struct devlink_resource *resource;
8694         int err = 0;
8695
8696         mutex_lock(&devlink->lock);
8697         resource = devlink_resource_find(devlink, NULL, resource_id);
8698         if (!resource) {
8699                 err = -EINVAL;
8700                 goto out;
8701         }
8702         *p_resource_size = resource->size_new;
8703         resource->size = resource->size_new;
8704 out:
8705         mutex_unlock(&devlink->lock);
8706         return err;
8707 }
8708 EXPORT_SYMBOL_GPL(devlink_resource_size_get);
8709
8710 /**
8711  *      devlink_dpipe_table_resource_set - set the resource id
8712  *
8713  *      @devlink: devlink
8714  *      @table_name: table name
8715  *      @resource_id: resource id
8716  *      @resource_units: number of resource's units consumed per table's entry
8717  */
8718 int devlink_dpipe_table_resource_set(struct devlink *devlink,
8719                                      const char *table_name, u64 resource_id,
8720                                      u64 resource_units)
8721 {
8722         struct devlink_dpipe_table *table;
8723         int err = 0;
8724
8725         mutex_lock(&devlink->lock);
8726         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
8727                                          table_name, devlink);
8728         if (!table) {
8729                 err = -EINVAL;
8730                 goto out;
8731         }
8732         table->resource_id = resource_id;
8733         table->resource_units = resource_units;
8734         table->resource_valid = true;
8735 out:
8736         mutex_unlock(&devlink->lock);
8737         return err;
8738 }
8739 EXPORT_SYMBOL_GPL(devlink_dpipe_table_resource_set);
8740
8741 /**
8742  *      devlink_resource_occ_get_register - register occupancy getter
8743  *
8744  *      @devlink: devlink
8745  *      @resource_id: resource id
8746  *      @occ_get: occupancy getter callback
8747  *      @occ_get_priv: occupancy getter callback priv
8748  */
8749 void devlink_resource_occ_get_register(struct devlink *devlink,
8750                                        u64 resource_id,
8751                                        devlink_resource_occ_get_t *occ_get,
8752                                        void *occ_get_priv)
8753 {
8754         struct devlink_resource *resource;
8755
8756         mutex_lock(&devlink->lock);
8757         resource = devlink_resource_find(devlink, NULL, resource_id);
8758         if (WARN_ON(!resource))
8759                 goto out;
8760         WARN_ON(resource->occ_get);
8761
8762         resource->occ_get = occ_get;
8763         resource->occ_get_priv = occ_get_priv;
8764 out:
8765         mutex_unlock(&devlink->lock);
8766 }
8767 EXPORT_SYMBOL_GPL(devlink_resource_occ_get_register);
8768
8769 /**
8770  *      devlink_resource_occ_get_unregister - unregister occupancy getter
8771  *
8772  *      @devlink: devlink
8773  *      @resource_id: resource id
8774  */
8775 void devlink_resource_occ_get_unregister(struct devlink *devlink,
8776                                          u64 resource_id)
8777 {
8778         struct devlink_resource *resource;
8779
8780         mutex_lock(&devlink->lock);
8781         resource = devlink_resource_find(devlink, NULL, resource_id);
8782         if (WARN_ON(!resource))
8783                 goto out;
8784         WARN_ON(!resource->occ_get);
8785
8786         resource->occ_get = NULL;
8787         resource->occ_get_priv = NULL;
8788 out:
8789         mutex_unlock(&devlink->lock);
8790 }
8791 EXPORT_SYMBOL_GPL(devlink_resource_occ_get_unregister);
8792
8793 static int devlink_param_verify(const struct devlink_param *param)
8794 {
8795         if (!param || !param->name || !param->supported_cmodes)
8796                 return -EINVAL;
8797         if (param->generic)
8798                 return devlink_param_generic_verify(param);
8799         else
8800                 return devlink_param_driver_verify(param);
8801 }
8802
8803 static int __devlink_params_register(struct devlink *devlink,
8804                                      unsigned int port_index,
8805                                      struct list_head *param_list,
8806                                      const struct devlink_param *params,
8807                                      size_t params_count,
8808                                      enum devlink_command reg_cmd,
8809                                      enum devlink_command unreg_cmd)
8810 {
8811         const struct devlink_param *param = params;
8812         int i;
8813         int err;
8814
8815         mutex_lock(&devlink->lock);
8816         for (i = 0; i < params_count; i++, param++) {
8817                 err = devlink_param_verify(param);
8818                 if (err)
8819                         goto rollback;
8820
8821                 err = devlink_param_register_one(devlink, port_index,
8822                                                  param_list, param, reg_cmd);
8823                 if (err)
8824                         goto rollback;
8825         }
8826
8827         mutex_unlock(&devlink->lock);
8828         return 0;
8829
8830 rollback:
8831         if (!i)
8832                 goto unlock;
8833         for (param--; i > 0; i--, param--)
8834                 devlink_param_unregister_one(devlink, port_index, param_list,
8835                                              param, unreg_cmd);
8836 unlock:
8837         mutex_unlock(&devlink->lock);
8838         return err;
8839 }
8840
8841 static void __devlink_params_unregister(struct devlink *devlink,
8842                                         unsigned int port_index,
8843                                         struct list_head *param_list,
8844                                         const struct devlink_param *params,
8845                                         size_t params_count,
8846                                         enum devlink_command cmd)
8847 {
8848         const struct devlink_param *param = params;
8849         int i;
8850
8851         mutex_lock(&devlink->lock);
8852         for (i = 0; i < params_count; i++, param++)
8853                 devlink_param_unregister_one(devlink, 0, param_list, param,
8854                                              cmd);
8855         mutex_unlock(&devlink->lock);
8856 }
8857
8858 /**
8859  *      devlink_params_register - register configuration parameters
8860  *
8861  *      @devlink: devlink
8862  *      @params: configuration parameters array
8863  *      @params_count: number of parameters provided
8864  *
8865  *      Register the configuration parameters supported by the driver.
8866  */
8867 int devlink_params_register(struct devlink *devlink,
8868                             const struct devlink_param *params,
8869                             size_t params_count)
8870 {
8871         return __devlink_params_register(devlink, 0, &devlink->param_list,
8872                                          params, params_count,
8873                                          DEVLINK_CMD_PARAM_NEW,
8874                                          DEVLINK_CMD_PARAM_DEL);
8875 }
8876 EXPORT_SYMBOL_GPL(devlink_params_register);
8877
8878 /**
8879  *      devlink_params_unregister - unregister configuration parameters
8880  *      @devlink: devlink
8881  *      @params: configuration parameters to unregister
8882  *      @params_count: number of parameters provided
8883  */
8884 void devlink_params_unregister(struct devlink *devlink,
8885                                const struct devlink_param *params,
8886                                size_t params_count)
8887 {
8888         return __devlink_params_unregister(devlink, 0, &devlink->param_list,
8889                                            params, params_count,
8890                                            DEVLINK_CMD_PARAM_DEL);
8891 }
8892 EXPORT_SYMBOL_GPL(devlink_params_unregister);
8893
8894 /**
8895  *      devlink_params_publish - publish configuration parameters
8896  *
8897  *      @devlink: devlink
8898  *
8899  *      Publish previously registered configuration parameters.
8900  */
8901 void devlink_params_publish(struct devlink *devlink)
8902 {
8903         struct devlink_param_item *param_item;
8904
8905         list_for_each_entry(param_item, &devlink->param_list, list) {
8906                 if (param_item->published)
8907                         continue;
8908                 param_item->published = true;
8909                 devlink_param_notify(devlink, 0, param_item,
8910                                      DEVLINK_CMD_PARAM_NEW);
8911         }
8912 }
8913 EXPORT_SYMBOL_GPL(devlink_params_publish);
8914
8915 /**
8916  *      devlink_params_unpublish - unpublish configuration parameters
8917  *
8918  *      @devlink: devlink
8919  *
8920  *      Unpublish previously registered configuration parameters.
8921  */
8922 void devlink_params_unpublish(struct devlink *devlink)
8923 {
8924         struct devlink_param_item *param_item;
8925
8926         list_for_each_entry(param_item, &devlink->param_list, list) {
8927                 if (!param_item->published)
8928                         continue;
8929                 param_item->published = false;
8930                 devlink_param_notify(devlink, 0, param_item,
8931                                      DEVLINK_CMD_PARAM_DEL);
8932         }
8933 }
8934 EXPORT_SYMBOL_GPL(devlink_params_unpublish);
8935
8936 /**
8937  *      devlink_port_params_register - register port configuration parameters
8938  *
8939  *      @devlink_port: devlink port
8940  *      @params: configuration parameters array
8941  *      @params_count: number of parameters provided
8942  *
8943  *      Register the configuration parameters supported by the port.
8944  */
8945 int devlink_port_params_register(struct devlink_port *devlink_port,
8946                                  const struct devlink_param *params,
8947                                  size_t params_count)
8948 {
8949         return __devlink_params_register(devlink_port->devlink,
8950                                          devlink_port->index,
8951                                          &devlink_port->param_list, params,
8952                                          params_count,
8953                                          DEVLINK_CMD_PORT_PARAM_NEW,
8954                                          DEVLINK_CMD_PORT_PARAM_DEL);
8955 }
8956 EXPORT_SYMBOL_GPL(devlink_port_params_register);
8957
8958 /**
8959  *      devlink_port_params_unregister - unregister port configuration
8960  *      parameters
8961  *
8962  *      @devlink_port: devlink port
8963  *      @params: configuration parameters array
8964  *      @params_count: number of parameters provided
8965  */
8966 void devlink_port_params_unregister(struct devlink_port *devlink_port,
8967                                     const struct devlink_param *params,
8968                                     size_t params_count)
8969 {
8970         return __devlink_params_unregister(devlink_port->devlink,
8971                                            devlink_port->index,
8972                                            &devlink_port->param_list,
8973                                            params, params_count,
8974                                            DEVLINK_CMD_PORT_PARAM_DEL);
8975 }
8976 EXPORT_SYMBOL_GPL(devlink_port_params_unregister);
8977
8978 static int
8979 __devlink_param_driverinit_value_get(struct list_head *param_list, u32 param_id,
8980                                      union devlink_param_value *init_val)
8981 {
8982         struct devlink_param_item *param_item;
8983
8984         param_item = devlink_param_find_by_id(param_list, param_id);
8985         if (!param_item)
8986                 return -EINVAL;
8987
8988         if (!param_item->driverinit_value_valid ||
8989             !devlink_param_cmode_is_supported(param_item->param,
8990                                               DEVLINK_PARAM_CMODE_DRIVERINIT))
8991                 return -EOPNOTSUPP;
8992
8993         if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
8994                 strcpy(init_val->vstr, param_item->driverinit_value.vstr);
8995         else
8996                 *init_val = param_item->driverinit_value;
8997
8998         return 0;
8999 }
9000
9001 static int
9002 __devlink_param_driverinit_value_set(struct devlink *devlink,
9003                                      unsigned int port_index,
9004                                      struct list_head *param_list, u32 param_id,
9005                                      union devlink_param_value init_val,
9006                                      enum devlink_command cmd)
9007 {
9008         struct devlink_param_item *param_item;
9009
9010         param_item = devlink_param_find_by_id(param_list, param_id);
9011         if (!param_item)
9012                 return -EINVAL;
9013
9014         if (!devlink_param_cmode_is_supported(param_item->param,
9015                                               DEVLINK_PARAM_CMODE_DRIVERINIT))
9016                 return -EOPNOTSUPP;
9017
9018         if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
9019                 strcpy(param_item->driverinit_value.vstr, init_val.vstr);
9020         else
9021                 param_item->driverinit_value = init_val;
9022         param_item->driverinit_value_valid = true;
9023
9024         devlink_param_notify(devlink, port_index, param_item, cmd);
9025         return 0;
9026 }
9027
9028 /**
9029  *      devlink_param_driverinit_value_get - get configuration parameter
9030  *                                           value for driver initializing
9031  *
9032  *      @devlink: devlink
9033  *      @param_id: parameter ID
9034  *      @init_val: value of parameter in driverinit configuration mode
9035  *
9036  *      This function should be used by the driver to get driverinit
9037  *      configuration for initialization after reload command.
9038  */
9039 int devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id,
9040                                        union devlink_param_value *init_val)
9041 {
9042         if (!devlink_reload_supported(devlink->ops))
9043                 return -EOPNOTSUPP;
9044
9045         return __devlink_param_driverinit_value_get(&devlink->param_list,
9046                                                     param_id, init_val);
9047 }
9048 EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_get);
9049
9050 /**
9051  *      devlink_param_driverinit_value_set - set value of configuration
9052  *                                           parameter for driverinit
9053  *                                           configuration mode
9054  *
9055  *      @devlink: devlink
9056  *      @param_id: parameter ID
9057  *      @init_val: value of parameter to set for driverinit configuration mode
9058  *
9059  *      This function should be used by the driver to set driverinit
9060  *      configuration mode default value.
9061  */
9062 int devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id,
9063                                        union devlink_param_value init_val)
9064 {
9065         return __devlink_param_driverinit_value_set(devlink, 0,
9066                                                     &devlink->param_list,
9067                                                     param_id, init_val,
9068                                                     DEVLINK_CMD_PARAM_NEW);
9069 }
9070 EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_set);
9071
9072 /**
9073  *      devlink_port_param_driverinit_value_get - get configuration parameter
9074  *                                              value for driver initializing
9075  *
9076  *      @devlink_port: devlink_port
9077  *      @param_id: parameter ID
9078  *      @init_val: value of parameter in driverinit configuration mode
9079  *
9080  *      This function should be used by the driver to get driverinit
9081  *      configuration for initialization after reload command.
9082  */
9083 int devlink_port_param_driverinit_value_get(struct devlink_port *devlink_port,
9084                                             u32 param_id,
9085                                             union devlink_param_value *init_val)
9086 {
9087         struct devlink *devlink = devlink_port->devlink;
9088
9089         if (!devlink_reload_supported(devlink->ops))
9090                 return -EOPNOTSUPP;
9091
9092         return __devlink_param_driverinit_value_get(&devlink_port->param_list,
9093                                                     param_id, init_val);
9094 }
9095 EXPORT_SYMBOL_GPL(devlink_port_param_driverinit_value_get);
9096
9097 /**
9098  *     devlink_port_param_driverinit_value_set - set value of configuration
9099  *                                               parameter for driverinit
9100  *                                               configuration mode
9101  *
9102  *     @devlink_port: devlink_port
9103  *     @param_id: parameter ID
9104  *     @init_val: value of parameter to set for driverinit configuration mode
9105  *
9106  *     This function should be used by the driver to set driverinit
9107  *     configuration mode default value.
9108  */
9109 int devlink_port_param_driverinit_value_set(struct devlink_port *devlink_port,
9110                                             u32 param_id,
9111                                             union devlink_param_value init_val)
9112 {
9113         return __devlink_param_driverinit_value_set(devlink_port->devlink,
9114                                                     devlink_port->index,
9115                                                     &devlink_port->param_list,
9116                                                     param_id, init_val,
9117                                                     DEVLINK_CMD_PORT_PARAM_NEW);
9118 }
9119 EXPORT_SYMBOL_GPL(devlink_port_param_driverinit_value_set);
9120
9121 /**
9122  *      devlink_param_value_changed - notify devlink on a parameter's value
9123  *                                    change. Should be called by the driver
9124  *                                    right after the change.
9125  *
9126  *      @devlink: devlink
9127  *      @param_id: parameter ID
9128  *
9129  *      This function should be used by the driver to notify devlink on value
9130  *      change, excluding driverinit configuration mode.
9131  *      For driverinit configuration mode driver should use the function
9132  */
9133 void devlink_param_value_changed(struct devlink *devlink, u32 param_id)
9134 {
9135         struct devlink_param_item *param_item;
9136
9137         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
9138         WARN_ON(!param_item);
9139
9140         devlink_param_notify(devlink, 0, param_item, DEVLINK_CMD_PARAM_NEW);
9141 }
9142 EXPORT_SYMBOL_GPL(devlink_param_value_changed);
9143
9144 /**
9145  *     devlink_port_param_value_changed - notify devlink on a parameter's value
9146  *                                      change. Should be called by the driver
9147  *                                      right after the change.
9148  *
9149  *     @devlink_port: devlink_port
9150  *     @param_id: parameter ID
9151  *
9152  *     This function should be used by the driver to notify devlink on value
9153  *     change, excluding driverinit configuration mode.
9154  *     For driverinit configuration mode driver should use the function
9155  *     devlink_port_param_driverinit_value_set() instead.
9156  */
9157 void devlink_port_param_value_changed(struct devlink_port *devlink_port,
9158                                       u32 param_id)
9159 {
9160         struct devlink_param_item *param_item;
9161
9162         param_item = devlink_param_find_by_id(&devlink_port->param_list,
9163                                               param_id);
9164         WARN_ON(!param_item);
9165
9166         devlink_param_notify(devlink_port->devlink, devlink_port->index,
9167                              param_item, DEVLINK_CMD_PORT_PARAM_NEW);
9168 }
9169 EXPORT_SYMBOL_GPL(devlink_port_param_value_changed);
9170
9171 /**
9172  *      devlink_param_value_str_fill - Safely fill-up the string preventing
9173  *                                     from overflow of the preallocated buffer
9174  *
9175  *      @dst_val: destination devlink_param_value
9176  *      @src: source buffer
9177  */
9178 void devlink_param_value_str_fill(union devlink_param_value *dst_val,
9179                                   const char *src)
9180 {
9181         size_t len;
9182
9183         len = strlcpy(dst_val->vstr, src, __DEVLINK_PARAM_MAX_STRING_VALUE);
9184         WARN_ON(len >= __DEVLINK_PARAM_MAX_STRING_VALUE);
9185 }
9186 EXPORT_SYMBOL_GPL(devlink_param_value_str_fill);
9187
9188 /**
9189  *      devlink_region_create - create a new address region
9190  *
9191  *      @devlink: devlink
9192  *      @ops: region operations and name
9193  *      @region_max_snapshots: Maximum supported number of snapshots for region
9194  *      @region_size: size of region
9195  */
9196 struct devlink_region *
9197 devlink_region_create(struct devlink *devlink,
9198                       const struct devlink_region_ops *ops,
9199                       u32 region_max_snapshots, u64 region_size)
9200 {
9201         struct devlink_region *region;
9202         int err = 0;
9203
9204         if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
9205                 return ERR_PTR(-EINVAL);
9206
9207         mutex_lock(&devlink->lock);
9208
9209         if (devlink_region_get_by_name(devlink, ops->name)) {
9210                 err = -EEXIST;
9211                 goto unlock;
9212         }
9213
9214         region = kzalloc(sizeof(*region), GFP_KERNEL);
9215         if (!region) {
9216                 err = -ENOMEM;
9217                 goto unlock;
9218         }
9219
9220         region->devlink = devlink;
9221         region->max_snapshots = region_max_snapshots;
9222         region->ops = ops;
9223         region->size = region_size;
9224         INIT_LIST_HEAD(&region->snapshot_list);
9225         list_add_tail(&region->list, &devlink->region_list);
9226         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
9227
9228         mutex_unlock(&devlink->lock);
9229         return region;
9230
9231 unlock:
9232         mutex_unlock(&devlink->lock);
9233         return ERR_PTR(err);
9234 }
9235 EXPORT_SYMBOL_GPL(devlink_region_create);
9236
9237 /**
9238  *      devlink_port_region_create - create a new address region for a port
9239  *
9240  *      @port: devlink port
9241  *      @ops: region operations and name
9242  *      @region_max_snapshots: Maximum supported number of snapshots for region
9243  *      @region_size: size of region
9244  */
9245 struct devlink_region *
9246 devlink_port_region_create(struct devlink_port *port,
9247                            const struct devlink_port_region_ops *ops,
9248                            u32 region_max_snapshots, u64 region_size)
9249 {
9250         struct devlink *devlink = port->devlink;
9251         struct devlink_region *region;
9252         int err = 0;
9253
9254         if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
9255                 return ERR_PTR(-EINVAL);
9256
9257         mutex_lock(&devlink->lock);
9258
9259         if (devlink_port_region_get_by_name(port, ops->name)) {
9260                 err = -EEXIST;
9261                 goto unlock;
9262         }
9263
9264         region = kzalloc(sizeof(*region), GFP_KERNEL);
9265         if (!region) {
9266                 err = -ENOMEM;
9267                 goto unlock;
9268         }
9269
9270         region->devlink = devlink;
9271         region->port = port;
9272         region->max_snapshots = region_max_snapshots;
9273         region->port_ops = ops;
9274         region->size = region_size;
9275         INIT_LIST_HEAD(&region->snapshot_list);
9276         list_add_tail(&region->list, &port->region_list);
9277         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
9278
9279         mutex_unlock(&devlink->lock);
9280         return region;
9281
9282 unlock:
9283         mutex_unlock(&devlink->lock);
9284         return ERR_PTR(err);
9285 }
9286 EXPORT_SYMBOL_GPL(devlink_port_region_create);
9287
9288 /**
9289  *      devlink_region_destroy - destroy address region
9290  *
9291  *      @region: devlink region to destroy
9292  */
9293 void devlink_region_destroy(struct devlink_region *region)
9294 {
9295         struct devlink *devlink = region->devlink;
9296         struct devlink_snapshot *snapshot, *ts;
9297
9298         mutex_lock(&devlink->lock);
9299
9300         /* Free all snapshots of region */
9301         list_for_each_entry_safe(snapshot, ts, &region->snapshot_list, list)
9302                 devlink_region_snapshot_del(region, snapshot);
9303
9304         list_del(&region->list);
9305
9306         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
9307         mutex_unlock(&devlink->lock);
9308         kfree(region);
9309 }
9310 EXPORT_SYMBOL_GPL(devlink_region_destroy);
9311
9312 /**
9313  *      devlink_region_snapshot_id_get - get snapshot ID
9314  *
9315  *      This callback should be called when adding a new snapshot,
9316  *      Driver should use the same id for multiple snapshots taken
9317  *      on multiple regions at the same time/by the same trigger.
9318  *
9319  *      The caller of this function must use devlink_region_snapshot_id_put
9320  *      when finished creating regions using this id.
9321  *
9322  *      Returns zero on success, or a negative error code on failure.
9323  *
9324  *      @devlink: devlink
9325  *      @id: storage to return id
9326  */
9327 int devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
9328 {
9329         int err;
9330
9331         mutex_lock(&devlink->lock);
9332         err = __devlink_region_snapshot_id_get(devlink, id);
9333         mutex_unlock(&devlink->lock);
9334
9335         return err;
9336 }
9337 EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_get);
9338
9339 /**
9340  *      devlink_region_snapshot_id_put - put snapshot ID reference
9341  *
9342  *      This should be called by a driver after finishing creating snapshots
9343  *      with an id. Doing so ensures that the ID can later be released in the
9344  *      event that all snapshots using it have been destroyed.
9345  *
9346  *      @devlink: devlink
9347  *      @id: id to release reference on
9348  */
9349 void devlink_region_snapshot_id_put(struct devlink *devlink, u32 id)
9350 {
9351         mutex_lock(&devlink->lock);
9352         __devlink_snapshot_id_decrement(devlink, id);
9353         mutex_unlock(&devlink->lock);
9354 }
9355 EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_put);
9356
9357 /**
9358  *      devlink_region_snapshot_create - create a new snapshot
9359  *      This will add a new snapshot of a region. The snapshot
9360  *      will be stored on the region struct and can be accessed
9361  *      from devlink. This is useful for future analyses of snapshots.
9362  *      Multiple snapshots can be created on a region.
9363  *      The @snapshot_id should be obtained using the getter function.
9364  *
9365  *      @region: devlink region of the snapshot
9366  *      @data: snapshot data
9367  *      @snapshot_id: snapshot id to be created
9368  */
9369 int devlink_region_snapshot_create(struct devlink_region *region,
9370                                    u8 *data, u32 snapshot_id)
9371 {
9372         struct devlink *devlink = region->devlink;
9373         int err;
9374
9375         mutex_lock(&devlink->lock);
9376         err = __devlink_region_snapshot_create(region, data, snapshot_id);
9377         mutex_unlock(&devlink->lock);
9378
9379         return err;
9380 }
9381 EXPORT_SYMBOL_GPL(devlink_region_snapshot_create);
9382
9383 #define DEVLINK_TRAP(_id, _type)                                              \
9384         {                                                                     \
9385                 .type = DEVLINK_TRAP_TYPE_##_type,                            \
9386                 .id = DEVLINK_TRAP_GENERIC_ID_##_id,                          \
9387                 .name = DEVLINK_TRAP_GENERIC_NAME_##_id,                      \
9388         }
9389
9390 static const struct devlink_trap devlink_trap_generic[] = {
9391         DEVLINK_TRAP(SMAC_MC, DROP),
9392         DEVLINK_TRAP(VLAN_TAG_MISMATCH, DROP),
9393         DEVLINK_TRAP(INGRESS_VLAN_FILTER, DROP),
9394         DEVLINK_TRAP(INGRESS_STP_FILTER, DROP),
9395         DEVLINK_TRAP(EMPTY_TX_LIST, DROP),
9396         DEVLINK_TRAP(PORT_LOOPBACK_FILTER, DROP),
9397         DEVLINK_TRAP(BLACKHOLE_ROUTE, DROP),
9398         DEVLINK_TRAP(TTL_ERROR, EXCEPTION),
9399         DEVLINK_TRAP(TAIL_DROP, DROP),
9400         DEVLINK_TRAP(NON_IP_PACKET, DROP),
9401         DEVLINK_TRAP(UC_DIP_MC_DMAC, DROP),
9402         DEVLINK_TRAP(DIP_LB, DROP),
9403         DEVLINK_TRAP(SIP_MC, DROP),
9404         DEVLINK_TRAP(SIP_LB, DROP),
9405         DEVLINK_TRAP(CORRUPTED_IP_HDR, DROP),
9406         DEVLINK_TRAP(IPV4_SIP_BC, DROP),
9407         DEVLINK_TRAP(IPV6_MC_DIP_RESERVED_SCOPE, DROP),
9408         DEVLINK_TRAP(IPV6_MC_DIP_INTERFACE_LOCAL_SCOPE, DROP),
9409         DEVLINK_TRAP(MTU_ERROR, EXCEPTION),
9410         DEVLINK_TRAP(UNRESOLVED_NEIGH, EXCEPTION),
9411         DEVLINK_TRAP(RPF, EXCEPTION),
9412         DEVLINK_TRAP(REJECT_ROUTE, EXCEPTION),
9413         DEVLINK_TRAP(IPV4_LPM_UNICAST_MISS, EXCEPTION),
9414         DEVLINK_TRAP(IPV6_LPM_UNICAST_MISS, EXCEPTION),
9415         DEVLINK_TRAP(NON_ROUTABLE, DROP),
9416         DEVLINK_TRAP(DECAP_ERROR, EXCEPTION),
9417         DEVLINK_TRAP(OVERLAY_SMAC_MC, DROP),
9418         DEVLINK_TRAP(INGRESS_FLOW_ACTION_DROP, DROP),
9419         DEVLINK_TRAP(EGRESS_FLOW_ACTION_DROP, DROP),
9420         DEVLINK_TRAP(STP, CONTROL),
9421         DEVLINK_TRAP(LACP, CONTROL),
9422         DEVLINK_TRAP(LLDP, CONTROL),
9423         DEVLINK_TRAP(IGMP_QUERY, CONTROL),
9424         DEVLINK_TRAP(IGMP_V1_REPORT, CONTROL),
9425         DEVLINK_TRAP(IGMP_V2_REPORT, CONTROL),
9426         DEVLINK_TRAP(IGMP_V3_REPORT, CONTROL),
9427         DEVLINK_TRAP(IGMP_V2_LEAVE, CONTROL),
9428         DEVLINK_TRAP(MLD_QUERY, CONTROL),
9429         DEVLINK_TRAP(MLD_V1_REPORT, CONTROL),
9430         DEVLINK_TRAP(MLD_V2_REPORT, CONTROL),
9431         DEVLINK_TRAP(MLD_V1_DONE, CONTROL),
9432         DEVLINK_TRAP(IPV4_DHCP, CONTROL),
9433         DEVLINK_TRAP(IPV6_DHCP, CONTROL),
9434         DEVLINK_TRAP(ARP_REQUEST, CONTROL),
9435         DEVLINK_TRAP(ARP_RESPONSE, CONTROL),
9436         DEVLINK_TRAP(ARP_OVERLAY, CONTROL),
9437         DEVLINK_TRAP(IPV6_NEIGH_SOLICIT, CONTROL),
9438         DEVLINK_TRAP(IPV6_NEIGH_ADVERT, CONTROL),
9439         DEVLINK_TRAP(IPV4_BFD, CONTROL),
9440         DEVLINK_TRAP(IPV6_BFD, CONTROL),
9441         DEVLINK_TRAP(IPV4_OSPF, CONTROL),
9442         DEVLINK_TRAP(IPV6_OSPF, CONTROL),
9443         DEVLINK_TRAP(IPV4_BGP, CONTROL),
9444         DEVLINK_TRAP(IPV6_BGP, CONTROL),
9445         DEVLINK_TRAP(IPV4_VRRP, CONTROL),
9446         DEVLINK_TRAP(IPV6_VRRP, CONTROL),
9447         DEVLINK_TRAP(IPV4_PIM, CONTROL),
9448         DEVLINK_TRAP(IPV6_PIM, CONTROL),
9449         DEVLINK_TRAP(UC_LB, CONTROL),
9450         DEVLINK_TRAP(LOCAL_ROUTE, CONTROL),
9451         DEVLINK_TRAP(EXTERNAL_ROUTE, CONTROL),
9452         DEVLINK_TRAP(IPV6_UC_DIP_LINK_LOCAL_SCOPE, CONTROL),
9453         DEVLINK_TRAP(IPV6_DIP_ALL_NODES, CONTROL),
9454         DEVLINK_TRAP(IPV6_DIP_ALL_ROUTERS, CONTROL),
9455         DEVLINK_TRAP(IPV6_ROUTER_SOLICIT, CONTROL),
9456         DEVLINK_TRAP(IPV6_ROUTER_ADVERT, CONTROL),
9457         DEVLINK_TRAP(IPV6_REDIRECT, CONTROL),
9458         DEVLINK_TRAP(IPV4_ROUTER_ALERT, CONTROL),
9459         DEVLINK_TRAP(IPV6_ROUTER_ALERT, CONTROL),
9460         DEVLINK_TRAP(PTP_EVENT, CONTROL),
9461         DEVLINK_TRAP(PTP_GENERAL, CONTROL),
9462         DEVLINK_TRAP(FLOW_ACTION_SAMPLE, CONTROL),
9463         DEVLINK_TRAP(FLOW_ACTION_TRAP, CONTROL),
9464         DEVLINK_TRAP(EARLY_DROP, DROP),
9465         DEVLINK_TRAP(VXLAN_PARSING, DROP),
9466         DEVLINK_TRAP(LLC_SNAP_PARSING, DROP),
9467         DEVLINK_TRAP(VLAN_PARSING, DROP),
9468         DEVLINK_TRAP(PPPOE_PPP_PARSING, DROP),
9469         DEVLINK_TRAP(MPLS_PARSING, DROP),
9470         DEVLINK_TRAP(ARP_PARSING, DROP),
9471         DEVLINK_TRAP(IP_1_PARSING, DROP),
9472         DEVLINK_TRAP(IP_N_PARSING, DROP),
9473         DEVLINK_TRAP(GRE_PARSING, DROP),
9474         DEVLINK_TRAP(UDP_PARSING, DROP),
9475         DEVLINK_TRAP(TCP_PARSING, DROP),
9476         DEVLINK_TRAP(IPSEC_PARSING, DROP),
9477         DEVLINK_TRAP(SCTP_PARSING, DROP),
9478         DEVLINK_TRAP(DCCP_PARSING, DROP),
9479         DEVLINK_TRAP(GTP_PARSING, DROP),
9480         DEVLINK_TRAP(ESP_PARSING, DROP),
9481 };
9482
9483 #define DEVLINK_TRAP_GROUP(_id)                                               \
9484         {                                                                     \
9485                 .id = DEVLINK_TRAP_GROUP_GENERIC_ID_##_id,                    \
9486                 .name = DEVLINK_TRAP_GROUP_GENERIC_NAME_##_id,                \
9487         }
9488
9489 static const struct devlink_trap_group devlink_trap_group_generic[] = {
9490         DEVLINK_TRAP_GROUP(L2_DROPS),
9491         DEVLINK_TRAP_GROUP(L3_DROPS),
9492         DEVLINK_TRAP_GROUP(L3_EXCEPTIONS),
9493         DEVLINK_TRAP_GROUP(BUFFER_DROPS),
9494         DEVLINK_TRAP_GROUP(TUNNEL_DROPS),
9495         DEVLINK_TRAP_GROUP(ACL_DROPS),
9496         DEVLINK_TRAP_GROUP(STP),
9497         DEVLINK_TRAP_GROUP(LACP),
9498         DEVLINK_TRAP_GROUP(LLDP),
9499         DEVLINK_TRAP_GROUP(MC_SNOOPING),
9500         DEVLINK_TRAP_GROUP(DHCP),
9501         DEVLINK_TRAP_GROUP(NEIGH_DISCOVERY),
9502         DEVLINK_TRAP_GROUP(BFD),
9503         DEVLINK_TRAP_GROUP(OSPF),
9504         DEVLINK_TRAP_GROUP(BGP),
9505         DEVLINK_TRAP_GROUP(VRRP),
9506         DEVLINK_TRAP_GROUP(PIM),
9507         DEVLINK_TRAP_GROUP(UC_LB),
9508         DEVLINK_TRAP_GROUP(LOCAL_DELIVERY),
9509         DEVLINK_TRAP_GROUP(EXTERNAL_DELIVERY),
9510         DEVLINK_TRAP_GROUP(IPV6),
9511         DEVLINK_TRAP_GROUP(PTP_EVENT),
9512         DEVLINK_TRAP_GROUP(PTP_GENERAL),
9513         DEVLINK_TRAP_GROUP(ACL_SAMPLE),
9514         DEVLINK_TRAP_GROUP(ACL_TRAP),
9515         DEVLINK_TRAP_GROUP(PARSER_ERROR_DROPS),
9516 };
9517
9518 static int devlink_trap_generic_verify(const struct devlink_trap *trap)
9519 {
9520         if (trap->id > DEVLINK_TRAP_GENERIC_ID_MAX)
9521                 return -EINVAL;
9522
9523         if (strcmp(trap->name, devlink_trap_generic[trap->id].name))
9524                 return -EINVAL;
9525
9526         if (trap->type != devlink_trap_generic[trap->id].type)
9527                 return -EINVAL;
9528
9529         return 0;
9530 }
9531
9532 static int devlink_trap_driver_verify(const struct devlink_trap *trap)
9533 {
9534         int i;
9535
9536         if (trap->id <= DEVLINK_TRAP_GENERIC_ID_MAX)
9537                 return -EINVAL;
9538
9539         for (i = 0; i < ARRAY_SIZE(devlink_trap_generic); i++) {
9540                 if (!strcmp(trap->name, devlink_trap_generic[i].name))
9541                         return -EEXIST;
9542         }
9543
9544         return 0;
9545 }
9546
9547 static int devlink_trap_verify(const struct devlink_trap *trap)
9548 {
9549         if (!trap || !trap->name)
9550                 return -EINVAL;
9551
9552         if (trap->generic)
9553                 return devlink_trap_generic_verify(trap);
9554         else
9555                 return devlink_trap_driver_verify(trap);
9556 }
9557
9558 static int
9559 devlink_trap_group_generic_verify(const struct devlink_trap_group *group)
9560 {
9561         if (group->id > DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
9562                 return -EINVAL;
9563
9564         if (strcmp(group->name, devlink_trap_group_generic[group->id].name))
9565                 return -EINVAL;
9566
9567         return 0;
9568 }
9569
9570 static int
9571 devlink_trap_group_driver_verify(const struct devlink_trap_group *group)
9572 {
9573         int i;
9574
9575         if (group->id <= DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
9576                 return -EINVAL;
9577
9578         for (i = 0; i < ARRAY_SIZE(devlink_trap_group_generic); i++) {
9579                 if (!strcmp(group->name, devlink_trap_group_generic[i].name))
9580                         return -EEXIST;
9581         }
9582
9583         return 0;
9584 }
9585
9586 static int devlink_trap_group_verify(const struct devlink_trap_group *group)
9587 {
9588         if (group->generic)
9589                 return devlink_trap_group_generic_verify(group);
9590         else
9591                 return devlink_trap_group_driver_verify(group);
9592 }
9593
9594 static void
9595 devlink_trap_group_notify(struct devlink *devlink,
9596                           const struct devlink_trap_group_item *group_item,
9597                           enum devlink_command cmd)
9598 {
9599         struct sk_buff *msg;
9600         int err;
9601
9602         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_GROUP_NEW &&
9603                      cmd != DEVLINK_CMD_TRAP_GROUP_DEL);
9604
9605         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
9606         if (!msg)
9607                 return;
9608
9609         err = devlink_nl_trap_group_fill(msg, devlink, group_item, cmd, 0, 0,
9610                                          0);
9611         if (err) {
9612                 nlmsg_free(msg);
9613                 return;
9614         }
9615
9616         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
9617                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
9618 }
9619
9620 static int
9621 devlink_trap_item_group_link(struct devlink *devlink,
9622                              struct devlink_trap_item *trap_item)
9623 {
9624         u16 group_id = trap_item->trap->init_group_id;
9625         struct devlink_trap_group_item *group_item;
9626
9627         group_item = devlink_trap_group_item_lookup_by_id(devlink, group_id);
9628         if (WARN_ON_ONCE(!group_item))
9629                 return -EINVAL;
9630
9631         trap_item->group_item = group_item;
9632
9633         return 0;
9634 }
9635
9636 static void devlink_trap_notify(struct devlink *devlink,
9637                                 const struct devlink_trap_item *trap_item,
9638                                 enum devlink_command cmd)
9639 {
9640         struct sk_buff *msg;
9641         int err;
9642
9643         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_NEW &&
9644                      cmd != DEVLINK_CMD_TRAP_DEL);
9645
9646         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
9647         if (!msg)
9648                 return;
9649
9650         err = devlink_nl_trap_fill(msg, devlink, trap_item, cmd, 0, 0, 0);
9651         if (err) {
9652                 nlmsg_free(msg);
9653                 return;
9654         }
9655
9656         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
9657                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
9658 }
9659
9660 static int
9661 devlink_trap_register(struct devlink *devlink,
9662                       const struct devlink_trap *trap, void *priv)
9663 {
9664         struct devlink_trap_item *trap_item;
9665         int err;
9666
9667         if (devlink_trap_item_lookup(devlink, trap->name))
9668                 return -EEXIST;
9669
9670         trap_item = kzalloc(sizeof(*trap_item), GFP_KERNEL);
9671         if (!trap_item)
9672                 return -ENOMEM;
9673
9674         trap_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
9675         if (!trap_item->stats) {
9676                 err = -ENOMEM;
9677                 goto err_stats_alloc;
9678         }
9679
9680         trap_item->trap = trap;
9681         trap_item->action = trap->init_action;
9682         trap_item->priv = priv;
9683
9684         err = devlink_trap_item_group_link(devlink, trap_item);
9685         if (err)
9686                 goto err_group_link;
9687
9688         err = devlink->ops->trap_init(devlink, trap, trap_item);
9689         if (err)
9690                 goto err_trap_init;
9691
9692         list_add_tail(&trap_item->list, &devlink->trap_list);
9693         devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
9694
9695         return 0;
9696
9697 err_trap_init:
9698 err_group_link:
9699         free_percpu(trap_item->stats);
9700 err_stats_alloc:
9701         kfree(trap_item);
9702         return err;
9703 }
9704
9705 static void devlink_trap_unregister(struct devlink *devlink,
9706                                     const struct devlink_trap *trap)
9707 {
9708         struct devlink_trap_item *trap_item;
9709
9710         trap_item = devlink_trap_item_lookup(devlink, trap->name);
9711         if (WARN_ON_ONCE(!trap_item))
9712                 return;
9713
9714         devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
9715         list_del(&trap_item->list);
9716         if (devlink->ops->trap_fini)
9717                 devlink->ops->trap_fini(devlink, trap, trap_item);
9718         free_percpu(trap_item->stats);
9719         kfree(trap_item);
9720 }
9721
9722 static void devlink_trap_disable(struct devlink *devlink,
9723                                  const struct devlink_trap *trap)
9724 {
9725         struct devlink_trap_item *trap_item;
9726
9727         trap_item = devlink_trap_item_lookup(devlink, trap->name);
9728         if (WARN_ON_ONCE(!trap_item))
9729                 return;
9730
9731         devlink->ops->trap_action_set(devlink, trap, DEVLINK_TRAP_ACTION_DROP,
9732                                       NULL);
9733         trap_item->action = DEVLINK_TRAP_ACTION_DROP;
9734 }
9735
9736 /**
9737  * devlink_traps_register - Register packet traps with devlink.
9738  * @devlink: devlink.
9739  * @traps: Packet traps.
9740  * @traps_count: Count of provided packet traps.
9741  * @priv: Driver private information.
9742  *
9743  * Return: Non-zero value on failure.
9744  */
9745 int devlink_traps_register(struct devlink *devlink,
9746                            const struct devlink_trap *traps,
9747                            size_t traps_count, void *priv)
9748 {
9749         int i, err;
9750
9751         if (!devlink->ops->trap_init || !devlink->ops->trap_action_set)
9752                 return -EINVAL;
9753
9754         mutex_lock(&devlink->lock);
9755         for (i = 0; i < traps_count; i++) {
9756                 const struct devlink_trap *trap = &traps[i];
9757
9758                 err = devlink_trap_verify(trap);
9759                 if (err)
9760                         goto err_trap_verify;
9761
9762                 err = devlink_trap_register(devlink, trap, priv);
9763                 if (err)
9764                         goto err_trap_register;
9765         }
9766         mutex_unlock(&devlink->lock);
9767
9768         return 0;
9769
9770 err_trap_register:
9771 err_trap_verify:
9772         for (i--; i >= 0; i--)
9773                 devlink_trap_unregister(devlink, &traps[i]);
9774         mutex_unlock(&devlink->lock);
9775         return err;
9776 }
9777 EXPORT_SYMBOL_GPL(devlink_traps_register);
9778
9779 /**
9780  * devlink_traps_unregister - Unregister packet traps from devlink.
9781  * @devlink: devlink.
9782  * @traps: Packet traps.
9783  * @traps_count: Count of provided packet traps.
9784  */
9785 void devlink_traps_unregister(struct devlink *devlink,
9786                               const struct devlink_trap *traps,
9787                               size_t traps_count)
9788 {
9789         int i;
9790
9791         mutex_lock(&devlink->lock);
9792         /* Make sure we do not have any packets in-flight while unregistering
9793          * traps by disabling all of them and waiting for a grace period.
9794          */
9795         for (i = traps_count - 1; i >= 0; i--)
9796                 devlink_trap_disable(devlink, &traps[i]);
9797         synchronize_rcu();
9798         for (i = traps_count - 1; i >= 0; i--)
9799                 devlink_trap_unregister(devlink, &traps[i]);
9800         mutex_unlock(&devlink->lock);
9801 }
9802 EXPORT_SYMBOL_GPL(devlink_traps_unregister);
9803
9804 static void
9805 devlink_trap_stats_update(struct devlink_stats __percpu *trap_stats,
9806                           size_t skb_len)
9807 {
9808         struct devlink_stats *stats;
9809
9810         stats = this_cpu_ptr(trap_stats);
9811         u64_stats_update_begin(&stats->syncp);
9812         stats->rx_bytes += skb_len;
9813         stats->rx_packets++;
9814         u64_stats_update_end(&stats->syncp);
9815 }
9816
9817 static void
9818 devlink_trap_report_metadata_set(struct devlink_trap_metadata *metadata,
9819                                  const struct devlink_trap_item *trap_item,
9820                                  struct devlink_port *in_devlink_port,
9821                                  const struct flow_action_cookie *fa_cookie)
9822 {
9823         metadata->trap_name = trap_item->trap->name;
9824         metadata->trap_group_name = trap_item->group_item->group->name;
9825         metadata->fa_cookie = fa_cookie;
9826         metadata->trap_type = trap_item->trap->type;
9827
9828         spin_lock(&in_devlink_port->type_lock);
9829         if (in_devlink_port->type == DEVLINK_PORT_TYPE_ETH)
9830                 metadata->input_dev = in_devlink_port->type_dev;
9831         spin_unlock(&in_devlink_port->type_lock);
9832 }
9833
9834 /**
9835  * devlink_trap_report - Report trapped packet to drop monitor.
9836  * @devlink: devlink.
9837  * @skb: Trapped packet.
9838  * @trap_ctx: Trap context.
9839  * @in_devlink_port: Input devlink port.
9840  * @fa_cookie: Flow action cookie. Could be NULL.
9841  */
9842 void devlink_trap_report(struct devlink *devlink, struct sk_buff *skb,
9843                          void *trap_ctx, struct devlink_port *in_devlink_port,
9844                          const struct flow_action_cookie *fa_cookie)
9845
9846 {
9847         struct devlink_trap_item *trap_item = trap_ctx;
9848
9849         devlink_trap_stats_update(trap_item->stats, skb->len);
9850         devlink_trap_stats_update(trap_item->group_item->stats, skb->len);
9851
9852         if (trace_devlink_trap_report_enabled()) {
9853                 struct devlink_trap_metadata metadata = {};
9854
9855                 devlink_trap_report_metadata_set(&metadata, trap_item,
9856                                                  in_devlink_port, fa_cookie);
9857                 trace_devlink_trap_report(devlink, skb, &metadata);
9858         }
9859 }
9860 EXPORT_SYMBOL_GPL(devlink_trap_report);
9861
9862 /**
9863  * devlink_trap_ctx_priv - Trap context to driver private information.
9864  * @trap_ctx: Trap context.
9865  *
9866  * Return: Driver private information passed during registration.
9867  */
9868 void *devlink_trap_ctx_priv(void *trap_ctx)
9869 {
9870         struct devlink_trap_item *trap_item = trap_ctx;
9871
9872         return trap_item->priv;
9873 }
9874 EXPORT_SYMBOL_GPL(devlink_trap_ctx_priv);
9875
9876 static int
9877 devlink_trap_group_item_policer_link(struct devlink *devlink,
9878                                      struct devlink_trap_group_item *group_item)
9879 {
9880         u32 policer_id = group_item->group->init_policer_id;
9881         struct devlink_trap_policer_item *policer_item;
9882
9883         if (policer_id == 0)
9884                 return 0;
9885
9886         policer_item = devlink_trap_policer_item_lookup(devlink, policer_id);
9887         if (WARN_ON_ONCE(!policer_item))
9888                 return -EINVAL;
9889
9890         group_item->policer_item = policer_item;
9891
9892         return 0;
9893 }
9894
9895 static int
9896 devlink_trap_group_register(struct devlink *devlink,
9897                             const struct devlink_trap_group *group)
9898 {
9899         struct devlink_trap_group_item *group_item;
9900         int err;
9901
9902         if (devlink_trap_group_item_lookup(devlink, group->name))
9903                 return -EEXIST;
9904
9905         group_item = kzalloc(sizeof(*group_item), GFP_KERNEL);
9906         if (!group_item)
9907                 return -ENOMEM;
9908
9909         group_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
9910         if (!group_item->stats) {
9911                 err = -ENOMEM;
9912                 goto err_stats_alloc;
9913         }
9914
9915         group_item->group = group;
9916
9917         err = devlink_trap_group_item_policer_link(devlink, group_item);
9918         if (err)
9919                 goto err_policer_link;
9920
9921         if (devlink->ops->trap_group_init) {
9922                 err = devlink->ops->trap_group_init(devlink, group);
9923                 if (err)
9924                         goto err_group_init;
9925         }
9926
9927         list_add_tail(&group_item->list, &devlink->trap_group_list);
9928         devlink_trap_group_notify(devlink, group_item,
9929                                   DEVLINK_CMD_TRAP_GROUP_NEW);
9930
9931         return 0;
9932
9933 err_group_init:
9934 err_policer_link:
9935         free_percpu(group_item->stats);
9936 err_stats_alloc:
9937         kfree(group_item);
9938         return err;
9939 }
9940
9941 static void
9942 devlink_trap_group_unregister(struct devlink *devlink,
9943                               const struct devlink_trap_group *group)
9944 {
9945         struct devlink_trap_group_item *group_item;
9946
9947         group_item = devlink_trap_group_item_lookup(devlink, group->name);
9948         if (WARN_ON_ONCE(!group_item))
9949                 return;
9950
9951         devlink_trap_group_notify(devlink, group_item,
9952                                   DEVLINK_CMD_TRAP_GROUP_DEL);
9953         list_del(&group_item->list);
9954         free_percpu(group_item->stats);
9955         kfree(group_item);
9956 }
9957
9958 /**
9959  * devlink_trap_groups_register - Register packet trap groups with devlink.
9960  * @devlink: devlink.
9961  * @groups: Packet trap groups.
9962  * @groups_count: Count of provided packet trap groups.
9963  *
9964  * Return: Non-zero value on failure.
9965  */
9966 int devlink_trap_groups_register(struct devlink *devlink,
9967                                  const struct devlink_trap_group *groups,
9968                                  size_t groups_count)
9969 {
9970         int i, err;
9971
9972         mutex_lock(&devlink->lock);
9973         for (i = 0; i < groups_count; i++) {
9974                 const struct devlink_trap_group *group = &groups[i];
9975
9976                 err = devlink_trap_group_verify(group);
9977                 if (err)
9978                         goto err_trap_group_verify;
9979
9980                 err = devlink_trap_group_register(devlink, group);
9981                 if (err)
9982                         goto err_trap_group_register;
9983         }
9984         mutex_unlock(&devlink->lock);
9985
9986         return 0;
9987
9988 err_trap_group_register:
9989 err_trap_group_verify:
9990         for (i--; i >= 0; i--)
9991                 devlink_trap_group_unregister(devlink, &groups[i]);
9992         mutex_unlock(&devlink->lock);
9993         return err;
9994 }
9995 EXPORT_SYMBOL_GPL(devlink_trap_groups_register);
9996
9997 /**
9998  * devlink_trap_groups_unregister - Unregister packet trap groups from devlink.
9999  * @devlink: devlink.
10000  * @groups: Packet trap groups.
10001  * @groups_count: Count of provided packet trap groups.
10002  */
10003 void devlink_trap_groups_unregister(struct devlink *devlink,
10004                                     const struct devlink_trap_group *groups,
10005                                     size_t groups_count)
10006 {
10007         int i;
10008
10009         mutex_lock(&devlink->lock);
10010         for (i = groups_count - 1; i >= 0; i--)
10011                 devlink_trap_group_unregister(devlink, &groups[i]);
10012         mutex_unlock(&devlink->lock);
10013 }
10014 EXPORT_SYMBOL_GPL(devlink_trap_groups_unregister);
10015
10016 static void
10017 devlink_trap_policer_notify(struct devlink *devlink,
10018                             const struct devlink_trap_policer_item *policer_item,
10019                             enum devlink_command cmd)
10020 {
10021         struct sk_buff *msg;
10022         int err;
10023
10024         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_POLICER_NEW &&
10025                      cmd != DEVLINK_CMD_TRAP_POLICER_DEL);
10026
10027         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
10028         if (!msg)
10029                 return;
10030
10031         err = devlink_nl_trap_policer_fill(msg, devlink, policer_item, cmd, 0,
10032                                            0, 0);
10033         if (err) {
10034                 nlmsg_free(msg);
10035                 return;
10036         }
10037
10038         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
10039                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
10040 }
10041
10042 static int
10043 devlink_trap_policer_register(struct devlink *devlink,
10044                               const struct devlink_trap_policer *policer)
10045 {
10046         struct devlink_trap_policer_item *policer_item;
10047         int err;
10048
10049         if (devlink_trap_policer_item_lookup(devlink, policer->id))
10050                 return -EEXIST;
10051
10052         policer_item = kzalloc(sizeof(*policer_item), GFP_KERNEL);
10053         if (!policer_item)
10054                 return -ENOMEM;
10055
10056         policer_item->policer = policer;
10057         policer_item->rate = policer->init_rate;
10058         policer_item->burst = policer->init_burst;
10059
10060         if (devlink->ops->trap_policer_init) {
10061                 err = devlink->ops->trap_policer_init(devlink, policer);
10062                 if (err)
10063                         goto err_policer_init;
10064         }
10065
10066         list_add_tail(&policer_item->list, &devlink->trap_policer_list);
10067         devlink_trap_policer_notify(devlink, policer_item,
10068                                     DEVLINK_CMD_TRAP_POLICER_NEW);
10069
10070         return 0;
10071
10072 err_policer_init:
10073         kfree(policer_item);
10074         return err;
10075 }
10076
10077 static void
10078 devlink_trap_policer_unregister(struct devlink *devlink,
10079                                 const struct devlink_trap_policer *policer)
10080 {
10081         struct devlink_trap_policer_item *policer_item;
10082
10083         policer_item = devlink_trap_policer_item_lookup(devlink, policer->id);
10084         if (WARN_ON_ONCE(!policer_item))
10085                 return;
10086
10087         devlink_trap_policer_notify(devlink, policer_item,
10088                                     DEVLINK_CMD_TRAP_POLICER_DEL);
10089         list_del(&policer_item->list);
10090         if (devlink->ops->trap_policer_fini)
10091                 devlink->ops->trap_policer_fini(devlink, policer);
10092         kfree(policer_item);
10093 }
10094
10095 /**
10096  * devlink_trap_policers_register - Register packet trap policers with devlink.
10097  * @devlink: devlink.
10098  * @policers: Packet trap policers.
10099  * @policers_count: Count of provided packet trap policers.
10100  *
10101  * Return: Non-zero value on failure.
10102  */
10103 int
10104 devlink_trap_policers_register(struct devlink *devlink,
10105                                const struct devlink_trap_policer *policers,
10106                                size_t policers_count)
10107 {
10108         int i, err;
10109
10110         mutex_lock(&devlink->lock);
10111         for (i = 0; i < policers_count; i++) {
10112                 const struct devlink_trap_policer *policer = &policers[i];
10113
10114                 if (WARN_ON(policer->id == 0 ||
10115                             policer->max_rate < policer->min_rate ||
10116                             policer->max_burst < policer->min_burst)) {
10117                         err = -EINVAL;
10118                         goto err_trap_policer_verify;
10119                 }
10120
10121                 err = devlink_trap_policer_register(devlink, policer);
10122                 if (err)
10123                         goto err_trap_policer_register;
10124         }
10125         mutex_unlock(&devlink->lock);
10126
10127         return 0;
10128
10129 err_trap_policer_register:
10130 err_trap_policer_verify:
10131         for (i--; i >= 0; i--)
10132                 devlink_trap_policer_unregister(devlink, &policers[i]);
10133         mutex_unlock(&devlink->lock);
10134         return err;
10135 }
10136 EXPORT_SYMBOL_GPL(devlink_trap_policers_register);
10137
10138 /**
10139  * devlink_trap_policers_unregister - Unregister packet trap policers from devlink.
10140  * @devlink: devlink.
10141  * @policers: Packet trap policers.
10142  * @policers_count: Count of provided packet trap policers.
10143  */
10144 void
10145 devlink_trap_policers_unregister(struct devlink *devlink,
10146                                  const struct devlink_trap_policer *policers,
10147                                  size_t policers_count)
10148 {
10149         int i;
10150
10151         mutex_lock(&devlink->lock);
10152         for (i = policers_count - 1; i >= 0; i--)
10153                 devlink_trap_policer_unregister(devlink, &policers[i]);
10154         mutex_unlock(&devlink->lock);
10155 }
10156 EXPORT_SYMBOL_GPL(devlink_trap_policers_unregister);
10157
10158 static void __devlink_compat_running_version(struct devlink *devlink,
10159                                              char *buf, size_t len)
10160 {
10161         const struct nlattr *nlattr;
10162         struct devlink_info_req req;
10163         struct sk_buff *msg;
10164         int rem, err;
10165
10166         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
10167         if (!msg)
10168                 return;
10169
10170         req.msg = msg;
10171         err = devlink->ops->info_get(devlink, &req, NULL);
10172         if (err)
10173                 goto free_msg;
10174
10175         nla_for_each_attr(nlattr, (void *)msg->data, msg->len, rem) {
10176                 const struct nlattr *kv;
10177                 int rem_kv;
10178
10179                 if (nla_type(nlattr) != DEVLINK_ATTR_INFO_VERSION_RUNNING)
10180                         continue;
10181
10182                 nla_for_each_nested(kv, nlattr, rem_kv) {
10183                         if (nla_type(kv) != DEVLINK_ATTR_INFO_VERSION_VALUE)
10184                                 continue;
10185
10186                         strlcat(buf, nla_data(kv), len);
10187                         strlcat(buf, " ", len);
10188                 }
10189         }
10190 free_msg:
10191         nlmsg_free(msg);
10192 }
10193
10194 void devlink_compat_running_version(struct net_device *dev,
10195                                     char *buf, size_t len)
10196 {
10197         struct devlink *devlink;
10198
10199         dev_hold(dev);
10200         rtnl_unlock();
10201
10202         devlink = netdev_to_devlink(dev);
10203         if (!devlink || !devlink->ops->info_get)
10204                 goto out;
10205
10206         mutex_lock(&devlink->lock);
10207         __devlink_compat_running_version(devlink, buf, len);
10208         mutex_unlock(&devlink->lock);
10209
10210 out:
10211         rtnl_lock();
10212         dev_put(dev);
10213 }
10214
10215 int devlink_compat_flash_update(struct net_device *dev, const char *file_name)
10216 {
10217         struct devlink_flash_update_params params = {};
10218         struct devlink *devlink;
10219         int ret;
10220
10221         dev_hold(dev);
10222         rtnl_unlock();
10223
10224         devlink = netdev_to_devlink(dev);
10225         if (!devlink || !devlink->ops->flash_update) {
10226                 ret = -EOPNOTSUPP;
10227                 goto out;
10228         }
10229
10230         params.file_name = file_name;
10231
10232         mutex_lock(&devlink->lock);
10233         ret = devlink->ops->flash_update(devlink, &params, NULL);
10234         mutex_unlock(&devlink->lock);
10235
10236 out:
10237         rtnl_lock();
10238         dev_put(dev);
10239
10240         return ret;
10241 }
10242
10243 int devlink_compat_phys_port_name_get(struct net_device *dev,
10244                                       char *name, size_t len)
10245 {
10246         struct devlink_port *devlink_port;
10247
10248         /* RTNL mutex is held here which ensures that devlink_port
10249          * instance cannot disappear in the middle. No need to take
10250          * any devlink lock as only permanent values are accessed.
10251          */
10252         ASSERT_RTNL();
10253
10254         devlink_port = netdev_to_devlink_port(dev);
10255         if (!devlink_port)
10256                 return -EOPNOTSUPP;
10257
10258         return __devlink_port_phys_port_name_get(devlink_port, name, len);
10259 }
10260
10261 int devlink_compat_switch_id_get(struct net_device *dev,
10262                                  struct netdev_phys_item_id *ppid)
10263 {
10264         struct devlink_port *devlink_port;
10265
10266         /* Caller must hold RTNL mutex or reference to dev, which ensures that
10267          * devlink_port instance cannot disappear in the middle. No need to take
10268          * any devlink lock as only permanent values are accessed.
10269          */
10270         devlink_port = netdev_to_devlink_port(dev);
10271         if (!devlink_port || !devlink_port->switch_port)
10272                 return -EOPNOTSUPP;
10273
10274         memcpy(ppid, &devlink_port->attrs.switch_id, sizeof(*ppid));
10275
10276         return 0;
10277 }
10278
10279 static void __net_exit devlink_pernet_pre_exit(struct net *net)
10280 {
10281         struct devlink *devlink;
10282         u32 actions_performed;
10283         int err;
10284
10285         /* In case network namespace is getting destroyed, reload
10286          * all devlink instances from this namespace into init_net.
10287          */
10288         mutex_lock(&devlink_mutex);
10289         list_for_each_entry(devlink, &devlink_list, list) {
10290                 if (net_eq(devlink_net(devlink), net)) {
10291                         if (WARN_ON(!devlink_reload_supported(devlink->ops)))
10292                                 continue;
10293                         err = devlink_reload(devlink, &init_net,
10294                                              DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
10295                                              DEVLINK_RELOAD_LIMIT_UNSPEC,
10296                                              &actions_performed, NULL);
10297                         if (err && err != -EOPNOTSUPP)
10298                                 pr_warn("Failed to reload devlink instance into init_net\n");
10299                 }
10300         }
10301         mutex_unlock(&devlink_mutex);
10302 }
10303
10304 static struct pernet_operations devlink_pernet_ops __net_initdata = {
10305         .pre_exit = devlink_pernet_pre_exit,
10306 };
10307
10308 static int __init devlink_init(void)
10309 {
10310         int err;
10311
10312         err = genl_register_family(&devlink_nl_family);
10313         if (err)
10314                 goto out;
10315         err = register_pernet_subsys(&devlink_pernet_ops);
10316
10317 out:
10318         WARN_ON(err);
10319         return err;
10320 }
10321
10322 subsys_initcall(devlink_init);