kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/mutex.h>
  11 #include <linux/slab.h>
  12 #include <linux/stacktrace.h>
  13 #include <linux/rculist.h>
  14 #include <linux/tracefs.h>
  15
  16 #include "tracing_map.h"
  17 #include "trace.h"
  18 #include "trace_dynevent.h"
  19
  20 #define SYNTH_SYSTEM            "synthetic"
  21 #define SYNTH_FIELDS_MAX        16
  22
  23 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  24
  25 struct hist_field;
  26
  27 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  28                                 struct tracing_map_elt *elt,
  29                                 struct ring_buffer_event *rbe,
  30                                 void *event);
  31
  32 #define HIST_FIELD_OPERANDS_MAX 2
  33 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  34 #define HIST_ACTIONS_MAX        8
  35
  36 enum field_op_id {
  37         FIELD_OP_NONE,
  38         FIELD_OP_PLUS,
  39         FIELD_OP_MINUS,
  40         FIELD_OP_UNARY_MINUS,
  41 };
  42
  43 /*
  44  * A hist_var (histogram variable) contains variable information for
  45  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
  46  * flag set.  A hist_var has a variable name e.g. ts0, and is
  47  * associated with a given histogram trigger, as specified by
  48  * hist_data.  The hist_var idx is the unique index assigned to the
  49  * variable by the hist trigger's tracing_map.  The idx is what is
  50  * used to set a variable's value and, by a variable reference, to
  51  * retrieve it.
  52  */
  53 struct hist_var {
  54         char                            *name;
  55         struct hist_trigger_data        *hist_data;
  56         unsigned int                    idx;
  57 };
  58
  59 struct hist_field {
  60         struct ftrace_event_field       *field;
  61         unsigned long                   flags;
  62         hist_field_fn_t                 fn;
  63         unsigned int                    size;
  64         unsigned int                    offset;
  65         unsigned int                    is_signed;
  66         const char                      *type;
  67         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
  68         struct hist_trigger_data        *hist_data;
  69
  70         /*
  71          * Variable fields contain variable-specific info in var.
  72          */
  73         struct hist_var                 var;
  74         enum field_op_id                operator;
  75         char                            *system;
  76         char                            *event_name;
  77
  78         /*
  79          * The name field is used for EXPR and VAR_REF fields.  VAR
  80          * fields contain the variable name in var.name.
  81          */
  82         char                            *name;
  83
  84         /*
  85          * When a histogram trigger is hit, if it has any references
  86          * to variables, the values of those variables are collected
  87          * into a var_ref_vals array by resolve_var_refs().  The
  88          * current value of each variable is read from the tracing_map
  89          * using the hist field's hist_var.idx and entered into the
  90          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
  91          */
  92         unsigned int                    var_ref_idx;
  93         bool                            read_once;
  94 };
  95
  96 static u64 hist_field_none(struct hist_field *field,
  97                            struct tracing_map_elt *elt,
  98                            struct ring_buffer_event *rbe,
  99                            void *event)
 100 {
 101         return 0;
 102 }
 103
 104 static u64 hist_field_counter(struct hist_field *field,
 105                               struct tracing_map_elt *elt,
 106                               struct ring_buffer_event *rbe,
 107                               void *event)
 108 {
 109         return 1;
 110 }
 111
 112 static u64 hist_field_string(struct hist_field *hist_field,
 113                              struct tracing_map_elt *elt,
 114                              struct ring_buffer_event *rbe,
 115                              void *event)
 116 {
 117         char *addr = (char *)(event + hist_field->field->offset);
 118
 119         return (u64)(unsigned long)addr;
 120 }
 121
 122 static u64 hist_field_dynstring(struct hist_field *hist_field,
 123                                 struct tracing_map_elt *elt,
 124                                 struct ring_buffer_event *rbe,
 125                                 void *event)
 126 {
 127         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 128         int str_loc = str_item & 0xffff;
 129         char *addr = (char *)(event + str_loc);
 130
 131         return (u64)(unsigned long)addr;
 132 }
 133
 134 static u64 hist_field_pstring(struct hist_field *hist_field,
 135                               struct tracing_map_elt *elt,
 136                               struct ring_buffer_event *rbe,
 137                               void *event)
 138 {
 139         char **addr = (char **)(event + hist_field->field->offset);
 140
 141         return (u64)(unsigned long)*addr;
 142 }
 143
 144 static u64 hist_field_log2(struct hist_field *hist_field,
 145                            struct tracing_map_elt *elt,
 146                            struct ring_buffer_event *rbe,
 147                            void *event)
 148 {
 149         struct hist_field *operand = hist_field->operands[0];
 150
 151         u64 val = operand->fn(operand, elt, rbe, event);
 152
 153         return (u64) ilog2(roundup_pow_of_two(val));
 154 }
 155
 156 static u64 hist_field_plus(struct hist_field *hist_field,
 157                            struct tracing_map_elt *elt,
 158                            struct ring_buffer_event *rbe,
 159                            void *event)
 160 {
 161         struct hist_field *operand1 = hist_field->operands[0];
 162         struct hist_field *operand2 = hist_field->operands[1];
 163
 164         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 165         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 166
 167         return val1 + val2;
 168 }
 169
 170 static u64 hist_field_minus(struct hist_field *hist_field,
 171                             struct tracing_map_elt *elt,
 172                             struct ring_buffer_event *rbe,
 173                             void *event)
 174 {
 175         struct hist_field *operand1 = hist_field->operands[0];
 176         struct hist_field *operand2 = hist_field->operands[1];
 177
 178         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 179         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 180
 181         return val1 - val2;
 182 }
 183
 184 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 185                                   struct tracing_map_elt *elt,
 186                                   struct ring_buffer_event *rbe,
 187                                   void *event)
 188 {
 189         struct hist_field *operand = hist_field->operands[0];
 190
 191         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 192         u64 val = (u64)-sval;
 193
 194         return val;
 195 }
 196
 197 #define DEFINE_HIST_FIELD_FN(type)                                      \
 198         static u64 hist_field_##type(struct hist_field *hist_field,     \
 199                                      struct tracing_map_elt *elt,       \
 200                                      struct ring_buffer_event *rbe,     \
 201                                      void *event)                       \
 202 {                                                                       \
 203         type *addr = (type *)(event + hist_field->field->offset);       \
 204                                                                         \
 205         return (u64)(unsigned long)*addr;                               \
 206 }
 207
 208 DEFINE_HIST_FIELD_FN(s64);
 209 DEFINE_HIST_FIELD_FN(u64);
 210 DEFINE_HIST_FIELD_FN(s32);
 211 DEFINE_HIST_FIELD_FN(u32);
 212 DEFINE_HIST_FIELD_FN(s16);
 213 DEFINE_HIST_FIELD_FN(u16);
 214 DEFINE_HIST_FIELD_FN(s8);
 215 DEFINE_HIST_FIELD_FN(u8);
 216
 217 #define for_each_hist_field(i, hist_data)       \
 218         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 219
 220 #define for_each_hist_val_field(i, hist_data)   \
 221         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 222
 223 #define for_each_hist_key_field(i, hist_data)   \
 224         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 225
 226 #define HIST_STACKTRACE_DEPTH   16
 227 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 228 #define HIST_STACKTRACE_SKIP    5
 229
 230 #define HITCOUNT_IDX            0
 231 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 232
 233 enum hist_field_flags {
 234         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 235         HIST_FIELD_FL_KEY               = 1 << 1,
 236         HIST_FIELD_FL_STRING            = 1 << 2,
 237         HIST_FIELD_FL_HEX               = 1 << 3,
 238         HIST_FIELD_FL_SYM               = 1 << 4,
 239         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 240         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 241         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 242         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 243         HIST_FIELD_FL_LOG2              = 1 << 9,
 244         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 245         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 246         HIST_FIELD_FL_VAR               = 1 << 12,
 247         HIST_FIELD_FL_EXPR              = 1 << 13,
 248         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 249         HIST_FIELD_FL_CPU               = 1 << 15,
 250         HIST_FIELD_FL_ALIAS             = 1 << 16,
 251 };
 252
 253 struct var_defs {
 254         unsigned int    n_vars;
 255         char            *name[TRACING_MAP_VARS_MAX];
 256         char            *expr[TRACING_MAP_VARS_MAX];
 257 };
 258
 259 struct hist_trigger_attrs {
 260         char            *keys_str;
 261         char            *vals_str;
 262         char            *sort_key_str;
 263         char            *name;
 264         char            *clock;
 265         bool            pause;
 266         bool            cont;
 267         bool            clear;
 268         bool            ts_in_usecs;
 269         unsigned int    map_bits;
 270
 271         char            *assignment_str[TRACING_MAP_VARS_MAX];
 272         unsigned int    n_assignments;
 273
 274         char            *action_str[HIST_ACTIONS_MAX];
 275         unsigned int    n_actions;
 276
 277         struct var_defs var_defs;
 278 };
 279
 280 struct field_var {
 281         struct hist_field       *var;
 282         struct hist_field       *val;
 283 };
 284
 285 struct field_var_hist {
 286         struct hist_trigger_data        *hist_data;
 287         char                            *cmd;
 288 };
 289
 290 struct hist_trigger_data {
 291         struct hist_field               *fields[HIST_FIELDS_MAX];
 292         unsigned int                    n_vals;
 293         unsigned int                    n_keys;
 294         unsigned int                    n_fields;
 295         unsigned int                    n_vars;
 296         unsigned int                    key_size;
 297         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 298         unsigned int                    n_sort_keys;
 299         struct trace_event_file         *event_file;
 300         struct hist_trigger_attrs       *attrs;
 301         struct tracing_map              *map;
 302         bool                            enable_timestamps;
 303         bool                            remove;
 304         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 305         unsigned int                    n_var_refs;
 306
 307         struct action_data              *actions[HIST_ACTIONS_MAX];
 308         unsigned int                    n_actions;
 309
 310         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 311         unsigned int                    n_field_vars;
 312         unsigned int                    n_field_var_str;
 313         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 314         unsigned int                    n_field_var_hists;
 315
 316         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 317         unsigned int                    n_save_vars;
 318         unsigned int                    n_save_var_str;
 319 };
 320
 321 static int synth_event_create(int argc, const char **argv);
 322 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 323 static int synth_event_release(struct dyn_event *ev);
 324 static bool synth_event_is_busy(struct dyn_event *ev);
 325 static bool synth_event_match(const char *system, const char *event,
 326                               struct dyn_event *ev);
 327
 328 static struct dyn_event_operations synth_event_ops = {
 329         .create = synth_event_create,
 330         .show = synth_event_show,
 331         .is_busy = synth_event_is_busy,
 332         .free = synth_event_release,
 333         .match = synth_event_match,
 334 };
 335
 336 struct synth_field {
 337         char *type;
 338         char *name;
 339         size_t size;
 340         bool is_signed;
 341         bool is_string;
 342 };
 343
 344 struct synth_event {
 345         struct dyn_event                        devent;
 346         int                                     ref;
 347         char                                    *name;
 348         struct synth_field                      **fields;
 349         unsigned int                            n_fields;
 350         unsigned int                            n_u64;
 351         struct trace_event_class                class;
 352         struct trace_event_call                 call;
 353         struct tracepoint                       *tp;
 354 };
 355
 356 static bool is_synth_event(struct dyn_event *ev)
 357 {
 358         return ev->ops == &synth_event_ops;
 359 }
 360
 361 static struct synth_event *to_synth_event(struct dyn_event *ev)
 362 {
 363         return container_of(ev, struct synth_event, devent);
 364 }
 365
 366 static bool synth_event_is_busy(struct dyn_event *ev)
 367 {
 368         struct synth_event *event = to_synth_event(ev);
 369
 370         return event->ref != 0;
 371 }
 372
 373 static bool synth_event_match(const char *system, const char *event,
 374                               struct dyn_event *ev)
 375 {
 376         struct synth_event *sev = to_synth_event(ev);
 377
 378         return strcmp(sev->name, event) == 0 &&
 379                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 380 }
 381
 382 struct action_data;
 383
 384 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 385                              struct tracing_map_elt *elt, void *rec,
 386                              struct ring_buffer_event *rbe, void *key,
 387                              struct action_data *data, u64 *var_ref_vals);
 388
 389 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 390
 391 enum handler_id {
 392         HANDLER_ONMATCH = 1,
 393         HANDLER_ONMAX,
 394         HANDLER_ONCHANGE,
 395 };
 396
 397 enum action_id {
 398         ACTION_SAVE = 1,
 399         ACTION_TRACE,
 400         ACTION_SNAPSHOT,
 401 };
 402
 403 struct action_data {
 404         enum handler_id         handler;
 405         enum action_id          action;
 406         char                    *action_name;
 407         action_fn_t             fn;
 408
 409         unsigned int            n_params;
 410         char                    *params[SYNTH_FIELDS_MAX];
 411
 412         /*
 413          * When a histogram trigger is hit, the values of any
 414          * references to variables, including variables being passed
 415          * as parameters to synthetic events, are collected into a
 416          * var_ref_vals array.  This var_ref_idx is the index of the
 417          * first param in the array to be passed to the synthetic
 418          * event invocation.
 419          */
 420         unsigned int            var_ref_idx;
 421         struct synth_event      *synth_event;
 422         bool                    use_trace_keyword;
 423         char                    *synth_event_name;
 424
 425         union {
 426                 struct {
 427                         char                    *event;
 428                         char                    *event_system;
 429                 } match_data;
 430
 431                 struct {
 432                         /*
 433                          * var_str contains the $-unstripped variable
 434                          * name referenced by var_ref, and used when
 435                          * printing the action.  Because var_ref
 436                          * creation is deferred to create_actions(),
 437                          * we need a per-action way to save it until
 438                          * then, thus var_str.
 439                          */
 440                         char                    *var_str;
 441
 442                         /*
 443                          * var_ref refers to the variable being
 444                          * tracked e.g onmax($var).
 445                          */
 446                         struct hist_field       *var_ref;
 447
 448                         /*
 449                          * track_var contains the 'invisible' tracking
 450                          * variable created to keep the current
 451                          * e.g. max value.
 452                          */
 453                         struct hist_field       *track_var;
 454
 455                         check_track_val_fn_t    check_val;
 456                         action_fn_t             save_data;
 457                 } track_data;
 458         };
 459 };
 460
 461 struct track_data {
 462         u64                             track_val;
 463         bool                            updated;
 464
 465         unsigned int                    key_len;
 466         void                            *key;
 467         struct tracing_map_elt          elt;
 468
 469         struct action_data              *action_data;
 470         struct hist_trigger_data        *hist_data;
 471 };
 472
 473 struct hist_elt_data {
 474         char *comm;
 475         u64 *var_ref_vals;
 476         char *field_var_str[SYNTH_FIELDS_MAX];
 477 };
 478
 479 struct snapshot_context {
 480         struct tracing_map_elt  *elt;
 481         void                    *key;
 482 };
 483
 484 static void track_data_free(struct track_data *track_data)
 485 {
 486         struct hist_elt_data *elt_data;
 487
 488         if (!track_data)
 489                 return;
 490
 491         kfree(track_data->key);
 492
 493         elt_data = track_data->elt.private_data;
 494         if (elt_data) {
 495                 kfree(elt_data->comm);
 496                 kfree(elt_data);
 497         }
 498
 499         kfree(track_data);
 500 }
 501
 502 static struct track_data *track_data_alloc(unsigned int key_len,
 503                                            struct action_data *action_data,
 504                                            struct hist_trigger_data *hist_data)
 505 {
 506         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 507         struct hist_elt_data *elt_data;
 508
 509         if (!data)
 510                 return ERR_PTR(-ENOMEM);
 511
 512         data->key = kzalloc(key_len, GFP_KERNEL);
 513         if (!data->key) {
 514                 track_data_free(data);
 515                 return ERR_PTR(-ENOMEM);
 516         }
 517
 518         data->key_len = key_len;
 519         data->action_data = action_data;
 520         data->hist_data = hist_data;
 521
 522         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 523         if (!elt_data) {
 524                 track_data_free(data);
 525                 return ERR_PTR(-ENOMEM);
 526         }
 527         data->elt.private_data = elt_data;
 528
 529         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 530         if (!elt_data->comm) {
 531                 track_data_free(data);
 532                 return ERR_PTR(-ENOMEM);
 533         }
 534
 535         return data;
 536 }
 537
 538 static char last_hist_cmd[MAX_FILTER_STR_VAL];
 539 static char hist_err_str[MAX_FILTER_STR_VAL];
 540
 541 static void last_cmd_set(char *str)
 542 {
 543         if (!str)
 544                 return;
 545
 546         strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
 547 }
 548
 549 static void hist_err(char *str, char *var)
 550 {
 551         int maxlen = MAX_FILTER_STR_VAL - 1;
 552
 553         if (!str)
 554                 return;
 555
 556         if (strlen(hist_err_str))
 557                 return;
 558
 559         if (!var)
 560                 var = "";
 561
 562         if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
 563                 return;
 564
 565         strcat(hist_err_str, str);
 566         strcat(hist_err_str, var);
 567 }
 568
 569 static void hist_err_event(char *str, char *system, char *event, char *var)
 570 {
 571         char err[MAX_FILTER_STR_VAL];
 572
 573         if (system && var)
 574                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
 575         else if (system)
 576                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
 577         else
 578                 strscpy(err, var, MAX_FILTER_STR_VAL);
 579
 580         hist_err(str, err);
 581 }
 582
 583 static void hist_err_clear(void)
 584 {
 585         hist_err_str[0] = '\0';
 586 }
 587
 588 static bool have_hist_err(void)
 589 {
 590         if (strlen(hist_err_str))
 591                 return true;
 592
 593         return false;
 594 }
 595
 596 struct synth_trace_event {
 597         struct trace_entry      ent;
 598         u64                     fields[];
 599 };
 600
 601 static int synth_event_define_fields(struct trace_event_call *call)
 602 {
 603         struct synth_trace_event trace;
 604         int offset = offsetof(typeof(trace), fields);
 605         struct synth_event *event = call->data;
 606         unsigned int i, size, n_u64;
 607         char *name, *type;
 608         bool is_signed;
 609         int ret = 0;
 610
 611         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 612                 size = event->fields[i]->size;
 613                 is_signed = event->fields[i]->is_signed;
 614                 type = event->fields[i]->type;
 615                 name = event->fields[i]->name;
 616                 ret = trace_define_field(call, type, name, offset, size,
 617                                          is_signed, FILTER_OTHER);
 618                 if (ret)
 619                         break;
 620
 621                 if (event->fields[i]->is_string) {
 622                         offset += STR_VAR_LEN_MAX;
 623                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 624                 } else {
 625                         offset += sizeof(u64);
 626                         n_u64++;
 627                 }
 628         }
 629
 630         event->n_u64 = n_u64;
 631
 632         return ret;
 633 }
 634
 635 static bool synth_field_signed(char *type)
 636 {
 637         if (str_has_prefix(type, "u"))
 638                 return false;
 639
 640         return true;
 641 }
 642
 643 static int synth_field_is_string(char *type)
 644 {
 645         if (strstr(type, "char[") != NULL)
 646                 return true;
 647
 648         return false;
 649 }
 650
 651 static int synth_field_string_size(char *type)
 652 {
 653         char buf[4], *end, *start;
 654         unsigned int len;
 655         int size, err;
 656
 657         start = strstr(type, "char[");
 658         if (start == NULL)
 659                 return -EINVAL;
 660         start += sizeof("char[") - 1;
 661
 662         end = strchr(type, ']');
 663         if (!end || end < start)
 664                 return -EINVAL;
 665
 666         len = end - start;
 667         if (len > 3)
 668                 return -EINVAL;
 669
 670         strncpy(buf, start, len);
 671         buf[len] = '\0';
 672
 673         err = kstrtouint(buf, 0, &size);
 674         if (err)
 675                 return err;
 676
 677         if (size > STR_VAR_LEN_MAX)
 678                 return -EINVAL;
 679
 680         return size;
 681 }
 682
 683 static int synth_field_size(char *type)
 684 {
 685         int size = 0;
 686
 687         if (strcmp(type, "s64") == 0)
 688                 size = sizeof(s64);
 689         else if (strcmp(type, "u64") == 0)
 690                 size = sizeof(u64);
 691         else if (strcmp(type, "s32") == 0)
 692                 size = sizeof(s32);
 693         else if (strcmp(type, "u32") == 0)
 694                 size = sizeof(u32);
 695         else if (strcmp(type, "s16") == 0)
 696                 size = sizeof(s16);
 697         else if (strcmp(type, "u16") == 0)
 698                 size = sizeof(u16);
 699         else if (strcmp(type, "s8") == 0)
 700                 size = sizeof(s8);
 701         else if (strcmp(type, "u8") == 0)
 702                 size = sizeof(u8);
 703         else if (strcmp(type, "char") == 0)
 704                 size = sizeof(char);
 705         else if (strcmp(type, "unsigned char") == 0)
 706                 size = sizeof(unsigned char);
 707         else if (strcmp(type, "int") == 0)
 708                 size = sizeof(int);
 709         else if (strcmp(type, "unsigned int") == 0)
 710                 size = sizeof(unsigned int);
 711         else if (strcmp(type, "long") == 0)
 712                 size = sizeof(long);
 713         else if (strcmp(type, "unsigned long") == 0)
 714                 size = sizeof(unsigned long);
 715         else if (strcmp(type, "pid_t") == 0)
 716                 size = sizeof(pid_t);
 717         else if (synth_field_is_string(type))
 718                 size = synth_field_string_size(type);
 719
 720         return size;
 721 }
 722
 723 static const char *synth_field_fmt(char *type)
 724 {
 725         const char *fmt = "%llu";
 726
 727         if (strcmp(type, "s64") == 0)
 728                 fmt = "%lld";
 729         else if (strcmp(type, "u64") == 0)
 730                 fmt = "%llu";
 731         else if (strcmp(type, "s32") == 0)
 732                 fmt = "%d";
 733         else if (strcmp(type, "u32") == 0)
 734                 fmt = "%u";
 735         else if (strcmp(type, "s16") == 0)
 736                 fmt = "%d";
 737         else if (strcmp(type, "u16") == 0)
 738                 fmt = "%u";
 739         else if (strcmp(type, "s8") == 0)
 740                 fmt = "%d";
 741         else if (strcmp(type, "u8") == 0)
 742                 fmt = "%u";
 743         else if (strcmp(type, "char") == 0)
 744                 fmt = "%d";
 745         else if (strcmp(type, "unsigned char") == 0)
 746                 fmt = "%u";
 747         else if (strcmp(type, "int") == 0)
 748                 fmt = "%d";
 749         else if (strcmp(type, "unsigned int") == 0)
 750                 fmt = "%u";
 751         else if (strcmp(type, "long") == 0)
 752                 fmt = "%ld";
 753         else if (strcmp(type, "unsigned long") == 0)
 754                 fmt = "%lu";
 755         else if (strcmp(type, "pid_t") == 0)
 756                 fmt = "%d";
 757         else if (synth_field_is_string(type))
 758                 fmt = "%s";
 759
 760         return fmt;
 761 }
 762
 763 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 764                                            int flags,
 765                                            struct trace_event *event)
 766 {
 767         struct trace_array *tr = iter->tr;
 768         struct trace_seq *s = &iter->seq;
 769         struct synth_trace_event *entry;
 770         struct synth_event *se;
 771         unsigned int i, n_u64;
 772         char print_fmt[32];
 773         const char *fmt;
 774
 775         entry = (struct synth_trace_event *)iter->ent;
 776         se = container_of(event, struct synth_event, call.event);
 777
 778         trace_seq_printf(s, "%s: ", se->name);
 779
 780         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 781                 if (trace_seq_has_overflowed(s))
 782                         goto end;
 783
 784                 fmt = synth_field_fmt(se->fields[i]->type);
 785
 786                 /* parameter types */
 787                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 788                         trace_seq_printf(s, "%s ", fmt);
 789
 790                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 791
 792                 /* parameter values */
 793                 if (se->fields[i]->is_string) {
 794                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 795                                          (char *)&entry->fields[n_u64],
 796                                          i == se->n_fields - 1 ? "" : " ");
 797                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 798                 } else {
 799                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 800                                          entry->fields[n_u64],
 801                                          i == se->n_fields - 1 ? "" : " ");
 802                         n_u64++;
 803                 }
 804         }
 805 end:
 806         trace_seq_putc(s, '\n');
 807
 808         return trace_handle_return(s);
 809 }
 810
 811 static struct trace_event_functions synth_event_funcs = {
 812         .trace          = print_synth_event
 813 };
 814
 815 static notrace void trace_event_raw_event_synth(void *__data,
 816                                                 u64 *var_ref_vals,
 817                                                 unsigned int var_ref_idx)
 818 {
 819         struct trace_event_file *trace_file = __data;
 820         struct synth_trace_event *entry;
 821         struct trace_event_buffer fbuffer;
 822         struct ring_buffer *buffer;
 823         struct synth_event *event;
 824         unsigned int i, n_u64;
 825         int fields_size = 0;
 826
 827         event = trace_file->event_call->data;
 828
 829         if (trace_trigger_soft_disabled(trace_file))
 830                 return;
 831
 832         fields_size = event->n_u64 * sizeof(u64);
 833
 834         /*
 835          * Avoid ring buffer recursion detection, as this event
 836          * is being performed within another event.
 837          */
 838         buffer = trace_file->tr->trace_buffer.buffer;
 839         ring_buffer_nest_start(buffer);
 840
 841         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 842                                            sizeof(*entry) + fields_size);
 843         if (!entry)
 844                 goto out;
 845
 846         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 847                 if (event->fields[i]->is_string) {
 848                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 849                         char *str_field = (char *)&entry->fields[n_u64];
 850
 851                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 852                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 853                 } else {
 854                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 855                         n_u64++;
 856                 }
 857         }
 858
 859         trace_event_buffer_commit(&fbuffer);
 860 out:
 861         ring_buffer_nest_end(buffer);
 862 }
 863
 864 static void free_synth_event_print_fmt(struct trace_event_call *call)
 865 {
 866         if (call) {
 867                 kfree(call->print_fmt);
 868                 call->print_fmt = NULL;
 869         }
 870 }
 871
 872 static int __set_synth_event_print_fmt(struct synth_event *event,
 873                                        char *buf, int len)
 874 {
 875         const char *fmt;
 876         int pos = 0;
 877         int i;
 878
 879         /* When len=0, we just calculate the needed length */
 880 #define LEN_OR_ZERO (len ? len - pos : 0)
 881
 882         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 883         for (i = 0; i < event->n_fields; i++) {
 884                 fmt = synth_field_fmt(event->fields[i]->type);
 885                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 886                                 event->fields[i]->name, fmt,
 887                                 i == event->n_fields - 1 ? "" : ", ");
 888         }
 889         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 890
 891         for (i = 0; i < event->n_fields; i++) {
 892                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 893                                 ", REC->%s", event->fields[i]->name);
 894         }
 895
 896 #undef LEN_OR_ZERO
 897
 898         /* return the length of print_fmt */
 899         return pos;
 900 }
 901
 902 static int set_synth_event_print_fmt(struct trace_event_call *call)
 903 {
 904         struct synth_event *event = call->data;
 905         char *print_fmt;
 906         int len;
 907
 908         /* First: called with 0 length to calculate the needed length */
 909         len = __set_synth_event_print_fmt(event, NULL, 0);
 910
 911         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 912         if (!print_fmt)
 913                 return -ENOMEM;
 914
 915         /* Second: actually write the @print_fmt */
 916         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 917         call->print_fmt = print_fmt;
 918
 919         return 0;
 920 }
 921
 922 static void free_synth_field(struct synth_field *field)
 923 {
 924         kfree(field->type);
 925         kfree(field->name);
 926         kfree(field);
 927 }
 928
 929 static struct synth_field *parse_synth_field(int argc, const char **argv,
 930                                              int *consumed)
 931 {
 932         struct synth_field *field;
 933         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
 934         int len, ret = 0;
 935
 936         if (field_type[0] == ';')
 937                 field_type++;
 938
 939         if (!strcmp(field_type, "unsigned")) {
 940                 if (argc < 3)
 941                         return ERR_PTR(-EINVAL);
 942                 prefix = "unsigned ";
 943                 field_type = argv[1];
 944                 field_name = argv[2];
 945                 *consumed = 3;
 946         } else {
 947                 field_name = argv[1];
 948                 *consumed = 2;
 949         }
 950
 951         field = kzalloc(sizeof(*field), GFP_KERNEL);
 952         if (!field)
 953                 return ERR_PTR(-ENOMEM);
 954
 955         len = strlen(field_name);
 956         array = strchr(field_name, '[');
 957         if (array)
 958                 len -= strlen(array);
 959         else if (field_name[len - 1] == ';')
 960                 len--;
 961
 962         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
 963         if (!field->name) {
 964                 ret = -ENOMEM;
 965                 goto free;
 966         }
 967
 968         if (field_type[0] == ';')
 969                 field_type++;
 970         len = strlen(field_type) + 1;
 971         if (array)
 972                 len += strlen(array);
 973         if (prefix)
 974                 len += strlen(prefix);
 975
 976         field->type = kzalloc(len, GFP_KERNEL);
 977         if (!field->type) {
 978                 ret = -ENOMEM;
 979                 goto free;
 980         }
 981         if (prefix)
 982                 strcat(field->type, prefix);
 983         strcat(field->type, field_type);
 984         if (array) {
 985                 strcat(field->type, array);
 986                 if (field->type[len - 1] == ';')
 987                         field->type[len - 1] = '\0';
 988         }
 989
 990         field->size = synth_field_size(field->type);
 991         if (!field->size) {
 992                 ret = -EINVAL;
 993                 goto free;
 994         }
 995
 996         if (synth_field_is_string(field->type))
 997                 field->is_string = true;
 998
 999         field->is_signed = synth_field_signed(field->type);
1000
1001  out:
1002         return field;
1003  free:
1004         free_synth_field(field);
1005         field = ERR_PTR(ret);
1006         goto out;
1007 }
1008
1009 static void free_synth_tracepoint(struct tracepoint *tp)
1010 {
1011         if (!tp)
1012                 return;
1013
1014         kfree(tp->name);
1015         kfree(tp);
1016 }
1017
1018 static struct tracepoint *alloc_synth_tracepoint(char *name)
1019 {
1020         struct tracepoint *tp;
1021
1022         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1023         if (!tp)
1024                 return ERR_PTR(-ENOMEM);
1025
1026         tp->name = kstrdup(name, GFP_KERNEL);
1027         if (!tp->name) {
1028                 kfree(tp);
1029                 return ERR_PTR(-ENOMEM);
1030         }
1031
1032         return tp;
1033 }
1034
1035 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1036                                     unsigned int var_ref_idx);
1037
1038 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1039                                unsigned int var_ref_idx)
1040 {
1041         struct tracepoint *tp = event->tp;
1042
1043         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1044                 struct tracepoint_func *probe_func_ptr;
1045                 synth_probe_func_t probe_func;
1046                 void *__data;
1047
1048                 if (!(cpu_online(raw_smp_processor_id())))
1049                         return;
1050
1051                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1052                 if (probe_func_ptr) {
1053                         do {
1054                                 probe_func = probe_func_ptr->func;
1055                                 __data = probe_func_ptr->data;
1056                                 probe_func(__data, var_ref_vals, var_ref_idx);
1057                         } while ((++probe_func_ptr)->func);
1058                 }
1059         }
1060 }
1061
1062 static struct synth_event *find_synth_event(const char *name)
1063 {
1064         struct dyn_event *pos;
1065         struct synth_event *event;
1066
1067         for_each_dyn_event(pos) {
1068                 if (!is_synth_event(pos))
1069                         continue;
1070                 event = to_synth_event(pos);
1071                 if (strcmp(event->name, name) == 0)
1072                         return event;
1073         }
1074
1075         return NULL;
1076 }
1077
1078 static int register_synth_event(struct synth_event *event)
1079 {
1080         struct trace_event_call *call = &event->call;
1081         int ret = 0;
1082
1083         event->call.class = &event->class;
1084         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1085         if (!event->class.system) {
1086                 ret = -ENOMEM;
1087                 goto out;
1088         }
1089
1090         event->tp = alloc_synth_tracepoint(event->name);
1091         if (IS_ERR(event->tp)) {
1092                 ret = PTR_ERR(event->tp);
1093                 event->tp = NULL;
1094                 goto out;
1095         }
1096
1097         INIT_LIST_HEAD(&call->class->fields);
1098         call->event.funcs = &synth_event_funcs;
1099         call->class->define_fields = synth_event_define_fields;
1100
1101         ret = register_trace_event(&call->event);
1102         if (!ret) {
1103                 ret = -ENODEV;
1104                 goto out;
1105         }
1106         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1107         call->class->reg = trace_event_reg;
1108         call->class->probe = trace_event_raw_event_synth;
1109         call->data = event;
1110         call->tp = event->tp;
1111
1112         ret = trace_add_event_call(call);
1113         if (ret) {
1114                 pr_warn("Failed to register synthetic event: %s\n",
1115                         trace_event_name(call));
1116                 goto err;
1117         }
1118
1119         ret = set_synth_event_print_fmt(call);
1120         if (ret < 0) {
1121                 trace_remove_event_call(call);
1122                 goto err;
1123         }
1124  out:
1125         return ret;
1126  err:
1127         unregister_trace_event(&call->event);
1128         goto out;
1129 }
1130
1131 static int unregister_synth_event(struct synth_event *event)
1132 {
1133         struct trace_event_call *call = &event->call;
1134         int ret;
1135
1136         ret = trace_remove_event_call(call);
1137
1138         return ret;
1139 }
1140
1141 static void free_synth_event(struct synth_event *event)
1142 {
1143         unsigned int i;
1144
1145         if (!event)
1146                 return;
1147
1148         for (i = 0; i < event->n_fields; i++)
1149                 free_synth_field(event->fields[i]);
1150
1151         kfree(event->fields);
1152         kfree(event->name);
1153         kfree(event->class.system);
1154         free_synth_tracepoint(event->tp);
1155         free_synth_event_print_fmt(&event->call);
1156         kfree(event);
1157 }
1158
1159 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1160                                              struct synth_field **fields)
1161 {
1162         struct synth_event *event;
1163         unsigned int i;
1164
1165         event = kzalloc(sizeof(*event), GFP_KERNEL);
1166         if (!event) {
1167                 event = ERR_PTR(-ENOMEM);
1168                 goto out;
1169         }
1170
1171         event->name = kstrdup(name, GFP_KERNEL);
1172         if (!event->name) {
1173                 kfree(event);
1174                 event = ERR_PTR(-ENOMEM);
1175                 goto out;
1176         }
1177
1178         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1179         if (!event->fields) {
1180                 free_synth_event(event);
1181                 event = ERR_PTR(-ENOMEM);
1182                 goto out;
1183         }
1184
1185         dyn_event_init(&event->devent, &synth_event_ops);
1186
1187         for (i = 0; i < n_fields; i++)
1188                 event->fields[i] = fields[i];
1189
1190         event->n_fields = n_fields;
1191  out:
1192         return event;
1193 }
1194
1195 static void action_trace(struct hist_trigger_data *hist_data,
1196                          struct tracing_map_elt *elt, void *rec,
1197                          struct ring_buffer_event *rbe, void *key,
1198                          struct action_data *data, u64 *var_ref_vals)
1199 {
1200         struct synth_event *event = data->synth_event;
1201
1202         trace_synth(event, var_ref_vals, data->var_ref_idx);
1203 }
1204
1205 struct hist_var_data {
1206         struct list_head list;
1207         struct hist_trigger_data *hist_data;
1208 };
1209
1210 static int __create_synth_event(int argc, const char *name, const char **argv)
1211 {
1212         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1213         struct synth_event *event = NULL;
1214         int i, consumed = 0, n_fields = 0, ret = 0;
1215
1216         /*
1217          * Argument syntax:
1218          *  - Add synthetic event: <event_name> field[;field] ...
1219          *  - Remove synthetic event: !<event_name> field[;field] ...
1220          *      where 'field' = type field_name
1221          */
1222
1223         if (name[0] == '\0' || argc < 1)
1224                 return -EINVAL;
1225
1226         mutex_lock(&event_mutex);
1227
1228         event = find_synth_event(name);
1229         if (event) {
1230                 ret = -EEXIST;
1231                 goto out;
1232         }
1233
1234         for (i = 0; i < argc - 1; i++) {
1235                 if (strcmp(argv[i], ";") == 0)
1236                         continue;
1237                 if (n_fields == SYNTH_FIELDS_MAX) {
1238                         ret = -EINVAL;
1239                         goto err;
1240                 }
1241
1242                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1243                 if (IS_ERR(field)) {
1244                         ret = PTR_ERR(field);
1245                         goto err;
1246                 }
1247                 fields[n_fields++] = field;
1248                 i += consumed - 1;
1249         }
1250
1251         if (i < argc && strcmp(argv[i], ";") != 0) {
1252                 ret = -EINVAL;
1253                 goto err;
1254         }
1255
1256         event = alloc_synth_event(name, n_fields, fields);
1257         if (IS_ERR(event)) {
1258                 ret = PTR_ERR(event);
1259                 event = NULL;
1260                 goto err;
1261         }
1262         ret = register_synth_event(event);
1263         if (!ret)
1264                 dyn_event_add(&event->devent);
1265         else
1266                 free_synth_event(event);
1267  out:
1268         mutex_unlock(&event_mutex);
1269
1270         return ret;
1271  err:
1272         for (i = 0; i < n_fields; i++)
1273                 free_synth_field(fields[i]);
1274
1275         goto out;
1276 }
1277
1278 static int create_or_delete_synth_event(int argc, char **argv)
1279 {
1280         const char *name = argv[0];
1281         struct synth_event *event = NULL;
1282         int ret;
1283
1284         /* trace_run_command() ensures argc != 0 */
1285         if (name[0] == '!') {
1286                 mutex_lock(&event_mutex);
1287                 event = find_synth_event(name + 1);
1288                 if (event) {
1289                         if (event->ref)
1290                                 ret = -EBUSY;
1291                         else {
1292                                 ret = unregister_synth_event(event);
1293                                 if (!ret) {
1294                                         dyn_event_remove(&event->devent);
1295                                         free_synth_event(event);
1296                                 }
1297                         }
1298                 } else
1299                         ret = -ENOENT;
1300                 mutex_unlock(&event_mutex);
1301                 return ret;
1302         }
1303
1304         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1305         return ret == -ECANCELED ? -EINVAL : ret;
1306 }
1307
1308 static int synth_event_create(int argc, const char **argv)
1309 {
1310         const char *name = argv[0];
1311         int len;
1312
1313         if (name[0] != 's' || name[1] != ':')
1314                 return -ECANCELED;
1315         name += 2;
1316
1317         /* This interface accepts group name prefix */
1318         if (strchr(name, '/')) {
1319                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
1320                 if (len == 0)
1321                         return -EINVAL;
1322                 name += len;
1323         }
1324         return __create_synth_event(argc - 1, name, argv + 1);
1325 }
1326
1327 static int synth_event_release(struct dyn_event *ev)
1328 {
1329         struct synth_event *event = to_synth_event(ev);
1330         int ret;
1331
1332         if (event->ref)
1333                 return -EBUSY;
1334
1335         ret = unregister_synth_event(event);
1336         if (ret)
1337                 return ret;
1338
1339         dyn_event_remove(ev);
1340         free_synth_event(event);
1341         return 0;
1342 }
1343
1344 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1345 {
1346         struct synth_field *field;
1347         unsigned int i;
1348
1349         seq_printf(m, "%s\t", event->name);
1350
1351         for (i = 0; i < event->n_fields; i++) {
1352                 field = event->fields[i];
1353
1354                 /* parameter values */
1355                 seq_printf(m, "%s %s%s", field->type, field->name,
1356                            i == event->n_fields - 1 ? "" : "; ");
1357         }
1358
1359         seq_putc(m, '\n');
1360
1361         return 0;
1362 }
1363
1364 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1365 {
1366         struct synth_event *event = to_synth_event(ev);
1367
1368         seq_printf(m, "s:%s/", event->class.system);
1369
1370         return __synth_event_show(m, event);
1371 }
1372
1373 static int synth_events_seq_show(struct seq_file *m, void *v)
1374 {
1375         struct dyn_event *ev = v;
1376
1377         if (!is_synth_event(ev))
1378                 return 0;
1379
1380         return __synth_event_show(m, to_synth_event(ev));
1381 }
1382
1383 static const struct seq_operations synth_events_seq_op = {
1384         .start  = dyn_event_seq_start,
1385         .next   = dyn_event_seq_next,
1386         .stop   = dyn_event_seq_stop,
1387         .show   = synth_events_seq_show,
1388 };
1389
1390 static int synth_events_open(struct inode *inode, struct file *file)
1391 {
1392         int ret;
1393
1394         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1395                 ret = dyn_events_release_all(&synth_event_ops);
1396                 if (ret < 0)
1397                         return ret;
1398         }
1399
1400         return seq_open(file, &synth_events_seq_op);
1401 }
1402
1403 static ssize_t synth_events_write(struct file *file,
1404                                   const char __user *buffer,
1405                                   size_t count, loff_t *ppos)
1406 {
1407         return trace_parse_run_command(file, buffer, count, ppos,
1408                                        create_or_delete_synth_event);
1409 }
1410
1411 static const struct file_operations synth_events_fops = {
1412         .open           = synth_events_open,
1413         .write          = synth_events_write,
1414         .read           = seq_read,
1415         .llseek         = seq_lseek,
1416         .release        = seq_release,
1417 };
1418
1419 static u64 hist_field_timestamp(struct hist_field *hist_field,
1420                                 struct tracing_map_elt *elt,
1421                                 struct ring_buffer_event *rbe,
1422                                 void *event)
1423 {
1424         struct hist_trigger_data *hist_data = hist_field->hist_data;
1425         struct trace_array *tr = hist_data->event_file->tr;
1426
1427         u64 ts = ring_buffer_event_time_stamp(rbe);
1428
1429         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1430                 ts = ns2usecs(ts);
1431
1432         return ts;
1433 }
1434
1435 static u64 hist_field_cpu(struct hist_field *hist_field,
1436                           struct tracing_map_elt *elt,
1437                           struct ring_buffer_event *rbe,
1438                           void *event)
1439 {
1440         int cpu = smp_processor_id();
1441
1442         return cpu;
1443 }
1444
1445 /**
1446  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1447  * @hist_field: The VAR_REF field to check
1448  * @var_data: The hist trigger that owns the variable
1449  * @var_idx: The trigger variable identifier
1450  *
1451  * Check the given VAR_REF field to see whether or not it references
1452  * the given variable associated with the given trigger.
1453  *
1454  * Return: The VAR_REF field if it does reference the variable, NULL if not
1455  */
1456 static struct hist_field *
1457 check_field_for_var_ref(struct hist_field *hist_field,
1458                         struct hist_trigger_data *var_data,
1459                         unsigned int var_idx)
1460 {
1461         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1462
1463         if (hist_field && hist_field->var.idx == var_idx &&
1464             hist_field->var.hist_data == var_data)
1465                 return hist_field;
1466
1467         return NULL;
1468 }
1469
1470 /**
1471  * find_var_ref - Check if a trigger has a reference to a trigger variable
1472  * @hist_data: The hist trigger that might have a reference to the variable
1473  * @var_data: The hist trigger that owns the variable
1474  * @var_idx: The trigger variable identifier
1475  *
1476  * Check the list of var_refs[] on the first hist trigger to see
1477  * whether any of them are references to the variable on the second
1478  * trigger.
1479  *
1480  * Return: The VAR_REF field referencing the variable if so, NULL if not
1481  */
1482 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1483                                        struct hist_trigger_data *var_data,
1484                                        unsigned int var_idx)
1485 {
1486         struct hist_field *hist_field;
1487         unsigned int i;
1488
1489         for (i = 0; i < hist_data->n_var_refs; i++) {
1490                 hist_field = hist_data->var_refs[i];
1491                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1492                         return hist_field;
1493         }
1494
1495         return NULL;
1496 }
1497
1498 /**
1499  * find_any_var_ref - Check if there is a reference to a given trigger variable
1500  * @hist_data: The hist trigger
1501  * @var_idx: The trigger variable identifier
1502  *
1503  * Check to see whether the given variable is currently referenced by
1504  * any other trigger.
1505  *
1506  * The trigger the variable is defined on is explicitly excluded - the
1507  * assumption being that a self-reference doesn't prevent a trigger
1508  * from being removed.
1509  *
1510  * Return: The VAR_REF field referencing the variable if so, NULL if not
1511  */
1512 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1513                                            unsigned int var_idx)
1514 {
1515         struct trace_array *tr = hist_data->event_file->tr;
1516         struct hist_field *found = NULL;
1517         struct hist_var_data *var_data;
1518
1519         list_for_each_entry(var_data, &tr->hist_vars, list) {
1520                 if (var_data->hist_data == hist_data)
1521                         continue;
1522                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1523                 if (found)
1524                         break;
1525         }
1526
1527         return found;
1528 }
1529
1530 /**
1531  * check_var_refs - Check if there is a reference to any of trigger's variables
1532  * @hist_data: The hist trigger
1533  *
1534  * A trigger can define one or more variables.  If any one of them is
1535  * currently referenced by any other trigger, this function will
1536  * determine that.
1537
1538  * Typically used to determine whether or not a trigger can be removed
1539  * - if there are any references to a trigger's variables, it cannot.
1540  *
1541  * Return: True if there is a reference to any of trigger's variables
1542  */
1543 static bool check_var_refs(struct hist_trigger_data *hist_data)
1544 {
1545         struct hist_field *field;
1546         bool found = false;
1547         int i;
1548
1549         for_each_hist_field(i, hist_data) {
1550                 field = hist_data->fields[i];
1551                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1552                         if (find_any_var_ref(hist_data, field->var.idx)) {
1553                                 found = true;
1554                                 break;
1555                         }
1556                 }
1557         }
1558
1559         return found;
1560 }
1561
1562 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1563 {
1564         struct trace_array *tr = hist_data->event_file->tr;
1565         struct hist_var_data *var_data, *found = NULL;
1566
1567         list_for_each_entry(var_data, &tr->hist_vars, list) {
1568                 if (var_data->hist_data == hist_data) {
1569                         found = var_data;
1570                         break;
1571                 }
1572         }
1573
1574         return found;
1575 }
1576
1577 static bool field_has_hist_vars(struct hist_field *hist_field,
1578                                 unsigned int level)
1579 {
1580         int i;
1581
1582         if (level > 3)
1583                 return false;
1584
1585         if (!hist_field)
1586                 return false;
1587
1588         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1589             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1590                 return true;
1591
1592         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1593                 struct hist_field *operand;
1594
1595                 operand = hist_field->operands[i];
1596                 if (field_has_hist_vars(operand, level + 1))
1597                         return true;
1598         }
1599
1600         return false;
1601 }
1602
1603 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1604 {
1605         struct hist_field *hist_field;
1606         int i;
1607
1608         for_each_hist_field(i, hist_data) {
1609                 hist_field = hist_data->fields[i];
1610                 if (field_has_hist_vars(hist_field, 0))
1611                         return true;
1612         }
1613
1614         return false;
1615 }
1616
1617 static int save_hist_vars(struct hist_trigger_data *hist_data)
1618 {
1619         struct trace_array *tr = hist_data->event_file->tr;
1620         struct hist_var_data *var_data;
1621
1622         var_data = find_hist_vars(hist_data);
1623         if (var_data)
1624                 return 0;
1625
1626         if (trace_array_get(tr) < 0)
1627                 return -ENODEV;
1628
1629         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1630         if (!var_data) {
1631                 trace_array_put(tr);
1632                 return -ENOMEM;
1633         }
1634
1635         var_data->hist_data = hist_data;
1636         list_add(&var_data->list, &tr->hist_vars);
1637
1638         return 0;
1639 }
1640
1641 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1642 {
1643         struct trace_array *tr = hist_data->event_file->tr;
1644         struct hist_var_data *var_data;
1645
1646         var_data = find_hist_vars(hist_data);
1647         if (!var_data)
1648                 return;
1649
1650         if (WARN_ON(check_var_refs(hist_data)))
1651                 return;
1652
1653         list_del(&var_data->list);
1654
1655         kfree(var_data);
1656
1657         trace_array_put(tr);
1658 }
1659
1660 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1661                                          const char *var_name)
1662 {
1663         struct hist_field *hist_field, *found = NULL;
1664         int i;
1665
1666         for_each_hist_field(i, hist_data) {
1667                 hist_field = hist_data->fields[i];
1668                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1669                     strcmp(hist_field->var.name, var_name) == 0) {
1670                         found = hist_field;
1671                         break;
1672                 }
1673         }
1674
1675         return found;
1676 }
1677
1678 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1679                                    struct trace_event_file *file,
1680                                    const char *var_name)
1681 {
1682         struct hist_trigger_data *test_data;
1683         struct event_trigger_data *test;
1684         struct hist_field *hist_field;
1685
1686         hist_field = find_var_field(hist_data, var_name);
1687         if (hist_field)
1688                 return hist_field;
1689
1690         list_for_each_entry_rcu(test, &file->triggers, list) {
1691                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1692                         test_data = test->private_data;
1693                         hist_field = find_var_field(test_data, var_name);
1694                         if (hist_field)
1695                                 return hist_field;
1696                 }
1697         }
1698
1699         return NULL;
1700 }
1701
1702 static struct trace_event_file *find_var_file(struct trace_array *tr,
1703                                               char *system,
1704                                               char *event_name,
1705                                               char *var_name)
1706 {
1707         struct hist_trigger_data *var_hist_data;
1708         struct hist_var_data *var_data;
1709         struct trace_event_file *file, *found = NULL;
1710
1711         if (system)
1712                 return find_event_file(tr, system, event_name);
1713
1714         list_for_each_entry(var_data, &tr->hist_vars, list) {
1715                 var_hist_data = var_data->hist_data;
1716                 file = var_hist_data->event_file;
1717                 if (file == found)
1718                         continue;
1719
1720                 if (find_var_field(var_hist_data, var_name)) {
1721                         if (found) {
1722                                 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1723                                 return NULL;
1724                         }
1725
1726                         found = file;
1727                 }
1728         }
1729
1730         return found;
1731 }
1732
1733 static struct hist_field *find_file_var(struct trace_event_file *file,
1734                                         const char *var_name)
1735 {
1736         struct hist_trigger_data *test_data;
1737         struct event_trigger_data *test;
1738         struct hist_field *hist_field;
1739
1740         list_for_each_entry_rcu(test, &file->triggers, list) {
1741                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1742                         test_data = test->private_data;
1743                         hist_field = find_var_field(test_data, var_name);
1744                         if (hist_field)
1745                                 return hist_field;
1746                 }
1747         }
1748
1749         return NULL;
1750 }
1751
1752 static struct hist_field *
1753 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1754 {
1755         struct trace_array *tr = hist_data->event_file->tr;
1756         struct hist_field *hist_field, *found = NULL;
1757         struct trace_event_file *file;
1758         unsigned int i;
1759
1760         for (i = 0; i < hist_data->n_actions; i++) {
1761                 struct action_data *data = hist_data->actions[i];
1762
1763                 if (data->handler == HANDLER_ONMATCH) {
1764                         char *system = data->match_data.event_system;
1765                         char *event_name = data->match_data.event;
1766
1767                         file = find_var_file(tr, system, event_name, var_name);
1768                         if (!file)
1769                                 continue;
1770                         hist_field = find_file_var(file, var_name);
1771                         if (hist_field) {
1772                                 if (found) {
1773                                         hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1774                                         return ERR_PTR(-EINVAL);
1775                                 }
1776
1777                                 found = hist_field;
1778                         }
1779                 }
1780         }
1781         return found;
1782 }
1783
1784 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1785                                          char *system,
1786                                          char *event_name,
1787                                          char *var_name)
1788 {
1789         struct trace_array *tr = hist_data->event_file->tr;
1790         struct hist_field *hist_field = NULL;
1791         struct trace_event_file *file;
1792
1793         if (!system || !event_name) {
1794                 hist_field = find_match_var(hist_data, var_name);
1795                 if (IS_ERR(hist_field))
1796                         return NULL;
1797                 if (hist_field)
1798                         return hist_field;
1799         }
1800
1801         file = find_var_file(tr, system, event_name, var_name);
1802         if (!file)
1803                 return NULL;
1804
1805         hist_field = find_file_var(file, var_name);
1806
1807         return hist_field;
1808 }
1809
1810 static u64 hist_field_var_ref(struct hist_field *hist_field,
1811                               struct tracing_map_elt *elt,
1812                               struct ring_buffer_event *rbe,
1813                               void *event)
1814 {
1815         struct hist_elt_data *elt_data;
1816         u64 var_val = 0;
1817
1818         elt_data = elt->private_data;
1819         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1820
1821         return var_val;
1822 }
1823
1824 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1825                              u64 *var_ref_vals, bool self)
1826 {
1827         struct hist_trigger_data *var_data;
1828         struct tracing_map_elt *var_elt;
1829         struct hist_field *hist_field;
1830         unsigned int i, var_idx;
1831         bool resolved = true;
1832         u64 var_val = 0;
1833
1834         for (i = 0; i < hist_data->n_var_refs; i++) {
1835                 hist_field = hist_data->var_refs[i];
1836                 var_idx = hist_field->var.idx;
1837                 var_data = hist_field->var.hist_data;
1838
1839                 if (var_data == NULL) {
1840                         resolved = false;
1841                         break;
1842                 }
1843
1844                 if ((self && var_data != hist_data) ||
1845                     (!self && var_data == hist_data))
1846                         continue;
1847
1848                 var_elt = tracing_map_lookup(var_data->map, key);
1849                 if (!var_elt) {
1850                         resolved = false;
1851                         break;
1852                 }
1853
1854                 if (!tracing_map_var_set(var_elt, var_idx)) {
1855                         resolved = false;
1856                         break;
1857                 }
1858
1859                 if (self || !hist_field->read_once)
1860                         var_val = tracing_map_read_var(var_elt, var_idx);
1861                 else
1862                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1863
1864                 var_ref_vals[i] = var_val;
1865         }
1866
1867         return resolved;
1868 }
1869
1870 static const char *hist_field_name(struct hist_field *field,
1871                                    unsigned int level)
1872 {
1873         const char *field_name = "";
1874
1875         if (level > 1)
1876                 return field_name;
1877
1878         if (field->field)
1879                 field_name = field->field->name;
1880         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1881                  field->flags & HIST_FIELD_FL_ALIAS)
1882                 field_name = hist_field_name(field->operands[0], ++level);
1883         else if (field->flags & HIST_FIELD_FL_CPU)
1884                 field_name = "cpu";
1885         else if (field->flags & HIST_FIELD_FL_EXPR ||
1886                  field->flags & HIST_FIELD_FL_VAR_REF) {
1887                 if (field->system) {
1888                         static char full_name[MAX_FILTER_STR_VAL];
1889
1890                         strcat(full_name, field->system);
1891                         strcat(full_name, ".");
1892                         strcat(full_name, field->event_name);
1893                         strcat(full_name, ".");
1894                         strcat(full_name, field->name);
1895                         field_name = full_name;
1896                 } else
1897                         field_name = field->name;
1898         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1899                 field_name = "common_timestamp";
1900
1901         if (field_name == NULL)
1902                 field_name = "";
1903
1904         return field_name;
1905 }
1906
1907 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1908 {
1909         hist_field_fn_t fn = NULL;
1910
1911         switch (field_size) {
1912         case 8:
1913                 if (field_is_signed)
1914                         fn = hist_field_s64;
1915                 else
1916                         fn = hist_field_u64;
1917                 break;
1918         case 4:
1919                 if (field_is_signed)
1920                         fn = hist_field_s32;
1921                 else
1922                         fn = hist_field_u32;
1923                 break;
1924         case 2:
1925                 if (field_is_signed)
1926                         fn = hist_field_s16;
1927                 else
1928                         fn = hist_field_u16;
1929                 break;
1930         case 1:
1931                 if (field_is_signed)
1932                         fn = hist_field_s8;
1933                 else
1934                         fn = hist_field_u8;
1935                 break;
1936         }
1937
1938         return fn;
1939 }
1940
1941 static int parse_map_size(char *str)
1942 {
1943         unsigned long size, map_bits;
1944         int ret;
1945
1946         strsep(&str, "=");
1947         if (!str) {
1948                 ret = -EINVAL;
1949                 goto out;
1950         }
1951
1952         ret = kstrtoul(str, 0, &size);
1953         if (ret)
1954                 goto out;
1955
1956         map_bits = ilog2(roundup_pow_of_two(size));
1957         if (map_bits < TRACING_MAP_BITS_MIN ||
1958             map_bits > TRACING_MAP_BITS_MAX)
1959                 ret = -EINVAL;
1960         else
1961                 ret = map_bits;
1962  out:
1963         return ret;
1964 }
1965
1966 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1967 {
1968         unsigned int i;
1969
1970         if (!attrs)
1971                 return;
1972
1973         for (i = 0; i < attrs->n_assignments; i++)
1974                 kfree(attrs->assignment_str[i]);
1975
1976         for (i = 0; i < attrs->n_actions; i++)
1977                 kfree(attrs->action_str[i]);
1978
1979         kfree(attrs->name);
1980         kfree(attrs->sort_key_str);
1981         kfree(attrs->keys_str);
1982         kfree(attrs->vals_str);
1983         kfree(attrs->clock);
1984         kfree(attrs);
1985 }
1986
1987 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1988 {
1989         int ret = -EINVAL;
1990
1991         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1992                 return ret;
1993
1994         if ((str_has_prefix(str, "onmatch(")) ||
1995             (str_has_prefix(str, "onmax(")) ||
1996             (str_has_prefix(str, "onchange("))) {
1997                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1998                 if (!attrs->action_str[attrs->n_actions]) {
1999                         ret = -ENOMEM;
2000                         return ret;
2001                 }
2002                 attrs->n_actions++;
2003                 ret = 0;
2004         }
2005
2006         return ret;
2007 }
2008
2009 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
2010 {
2011         int ret = 0;
2012
2013         if ((str_has_prefix(str, "key=")) ||
2014             (str_has_prefix(str, "keys="))) {
2015                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
2016                 if (!attrs->keys_str) {
2017                         ret = -ENOMEM;
2018                         goto out;
2019                 }
2020         } else if ((str_has_prefix(str, "val=")) ||
2021                    (str_has_prefix(str, "vals=")) ||
2022                    (str_has_prefix(str, "values="))) {
2023                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
2024                 if (!attrs->vals_str) {
2025                         ret = -ENOMEM;
2026                         goto out;
2027                 }
2028         } else if (str_has_prefix(str, "sort=")) {
2029                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
2030                 if (!attrs->sort_key_str) {
2031                         ret = -ENOMEM;
2032                         goto out;
2033                 }
2034         } else if (str_has_prefix(str, "name=")) {
2035                 attrs->name = kstrdup(str, GFP_KERNEL);
2036                 if (!attrs->name) {
2037                         ret = -ENOMEM;
2038                         goto out;
2039                 }
2040         } else if (str_has_prefix(str, "clock=")) {
2041                 strsep(&str, "=");
2042                 if (!str) {
2043                         ret = -EINVAL;
2044                         goto out;
2045                 }
2046
2047                 str = strstrip(str);
2048                 attrs->clock = kstrdup(str, GFP_KERNEL);
2049                 if (!attrs->clock) {
2050                         ret = -ENOMEM;
2051                         goto out;
2052                 }
2053         } else if (str_has_prefix(str, "size=")) {
2054                 int map_bits = parse_map_size(str);
2055
2056                 if (map_bits < 0) {
2057                         ret = map_bits;
2058                         goto out;
2059                 }
2060                 attrs->map_bits = map_bits;
2061         } else {
2062                 char *assignment;
2063
2064                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2065                         hist_err("Too many variables defined: ", str);
2066                         ret = -EINVAL;
2067                         goto out;
2068                 }
2069
2070                 assignment = kstrdup(str, GFP_KERNEL);
2071                 if (!assignment) {
2072                         ret = -ENOMEM;
2073                         goto out;
2074                 }
2075
2076                 attrs->assignment_str[attrs->n_assignments++] = assignment;
2077         }
2078  out:
2079         return ret;
2080 }
2081
2082 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
2083 {
2084         struct hist_trigger_attrs *attrs;
2085         int ret = 0;
2086
2087         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2088         if (!attrs)
2089                 return ERR_PTR(-ENOMEM);
2090
2091         while (trigger_str) {
2092                 char *str = strsep(&trigger_str, ":");
2093
2094                 if (strchr(str, '=')) {
2095                         ret = parse_assignment(str, attrs);
2096                         if (ret)
2097                                 goto free;
2098                 } else if (strcmp(str, "pause") == 0)
2099                         attrs->pause = true;
2100                 else if ((strcmp(str, "cont") == 0) ||
2101                          (strcmp(str, "continue") == 0))
2102                         attrs->cont = true;
2103                 else if (strcmp(str, "clear") == 0)
2104                         attrs->clear = true;
2105                 else {
2106                         ret = parse_action(str, attrs);
2107                         if (ret)
2108                                 goto free;
2109                 }
2110         }
2111
2112         if (!attrs->keys_str) {
2113                 ret = -EINVAL;
2114                 goto free;
2115         }
2116
2117         if (!attrs->clock) {
2118                 attrs->clock = kstrdup("global", GFP_KERNEL);
2119                 if (!attrs->clock) {
2120                         ret = -ENOMEM;
2121                         goto free;
2122                 }
2123         }
2124
2125         return attrs;
2126  free:
2127         destroy_hist_trigger_attrs(attrs);
2128
2129         return ERR_PTR(ret);
2130 }
2131
2132 static inline void save_comm(char *comm, struct task_struct *task)
2133 {
2134         if (!task->pid) {
2135                 strcpy(comm, "<idle>");
2136                 return;
2137         }
2138
2139         if (WARN_ON_ONCE(task->pid < 0)) {
2140                 strcpy(comm, "<XXX>");
2141                 return;
2142         }
2143
2144         strncpy(comm, task->comm, TASK_COMM_LEN);
2145 }
2146
2147 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2148 {
2149         unsigned int i;
2150
2151         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2152                 kfree(elt_data->field_var_str[i]);
2153
2154         kfree(elt_data->comm);
2155         kfree(elt_data);
2156 }
2157
2158 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2159 {
2160         struct hist_elt_data *elt_data = elt->private_data;
2161
2162         hist_elt_data_free(elt_data);
2163 }
2164
2165 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2166 {
2167         struct hist_trigger_data *hist_data = elt->map->private_data;
2168         unsigned int size = TASK_COMM_LEN;
2169         struct hist_elt_data *elt_data;
2170         struct hist_field *key_field;
2171         unsigned int i, n_str;
2172
2173         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2174         if (!elt_data)
2175                 return -ENOMEM;
2176
2177         for_each_hist_key_field(i, hist_data) {
2178                 key_field = hist_data->fields[i];
2179
2180                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2181                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2182                         if (!elt_data->comm) {
2183                                 kfree(elt_data);
2184                                 return -ENOMEM;
2185                         }
2186                         break;
2187                 }
2188         }
2189
2190         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2191
2192         size = STR_VAR_LEN_MAX;
2193
2194         for (i = 0; i < n_str; i++) {
2195                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2196                 if (!elt_data->field_var_str[i]) {
2197                         hist_elt_data_free(elt_data);
2198                         return -ENOMEM;
2199                 }
2200         }
2201
2202         elt->private_data = elt_data;
2203
2204         return 0;
2205 }
2206
2207 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2208 {
2209         struct hist_elt_data *elt_data = elt->private_data;
2210
2211         if (elt_data->comm)
2212                 save_comm(elt_data->comm, current);
2213 }
2214
2215 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2216         .elt_alloc      = hist_trigger_elt_data_alloc,
2217         .elt_free       = hist_trigger_elt_data_free,
2218         .elt_init       = hist_trigger_elt_data_init,
2219 };
2220
2221 static const char *get_hist_field_flags(struct hist_field *hist_field)
2222 {
2223         const char *flags_str = NULL;
2224
2225         if (hist_field->flags & HIST_FIELD_FL_HEX)
2226                 flags_str = "hex";
2227         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2228                 flags_str = "sym";
2229         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2230                 flags_str = "sym-offset";
2231         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2232                 flags_str = "execname";
2233         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2234                 flags_str = "syscall";
2235         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2236                 flags_str = "log2";
2237         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2238                 flags_str = "usecs";
2239
2240         return flags_str;
2241 }
2242
2243 static void expr_field_str(struct hist_field *field, char *expr)
2244 {
2245         if (field->flags & HIST_FIELD_FL_VAR_REF)
2246                 strcat(expr, "$");
2247
2248         strcat(expr, hist_field_name(field, 0));
2249
2250         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2251                 const char *flags_str = get_hist_field_flags(field);
2252
2253                 if (flags_str) {
2254                         strcat(expr, ".");
2255                         strcat(expr, flags_str);
2256                 }
2257         }
2258 }
2259
2260 static char *expr_str(struct hist_field *field, unsigned int level)
2261 {
2262         char *expr;
2263
2264         if (level > 1)
2265                 return NULL;
2266
2267         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2268         if (!expr)
2269                 return NULL;
2270
2271         if (!field->operands[0]) {
2272                 expr_field_str(field, expr);
2273                 return expr;
2274         }
2275
2276         if (field->operator == FIELD_OP_UNARY_MINUS) {
2277                 char *subexpr;
2278
2279                 strcat(expr, "-(");
2280                 subexpr = expr_str(field->operands[0], ++level);
2281                 if (!subexpr) {
2282                         kfree(expr);
2283                         return NULL;
2284                 }
2285                 strcat(expr, subexpr);
2286                 strcat(expr, ")");
2287
2288                 kfree(subexpr);
2289
2290                 return expr;
2291         }
2292
2293         expr_field_str(field->operands[0], expr);
2294
2295         switch (field->operator) {
2296         case FIELD_OP_MINUS:
2297                 strcat(expr, "-");
2298                 break;
2299         case FIELD_OP_PLUS:
2300                 strcat(expr, "+");
2301                 break;
2302         default:
2303                 kfree(expr);
2304                 return NULL;
2305         }
2306
2307         expr_field_str(field->operands[1], expr);
2308
2309         return expr;
2310 }
2311
2312 static int contains_operator(char *str)
2313 {
2314         enum field_op_id field_op = FIELD_OP_NONE;
2315         char *op;
2316
2317         op = strpbrk(str, "+-");
2318         if (!op)
2319                 return FIELD_OP_NONE;
2320
2321         switch (*op) {
2322         case '-':
2323                 if (*str == '-')
2324                         field_op = FIELD_OP_UNARY_MINUS;
2325                 else
2326                         field_op = FIELD_OP_MINUS;
2327                 break;
2328         case '+':
2329                 field_op = FIELD_OP_PLUS;
2330                 break;
2331         default:
2332                 break;
2333         }
2334
2335         return field_op;
2336 }
2337
2338 static void __destroy_hist_field(struct hist_field *hist_field)
2339 {
2340         kfree(hist_field->var.name);
2341         kfree(hist_field->name);
2342         kfree(hist_field->type);
2343
2344         kfree(hist_field);
2345 }
2346
2347 static void destroy_hist_field(struct hist_field *hist_field,
2348                                unsigned int level)
2349 {
2350         unsigned int i;
2351
2352         if (level > 3)
2353                 return;
2354
2355         if (!hist_field)
2356                 return;
2357
2358         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2359                 return; /* var refs will be destroyed separately */
2360
2361         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2362                 destroy_hist_field(hist_field->operands[i], level + 1);
2363
2364         __destroy_hist_field(hist_field);
2365 }
2366
2367 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2368                                             struct ftrace_event_field *field,
2369                                             unsigned long flags,
2370                                             char *var_name)
2371 {
2372         struct hist_field *hist_field;
2373
2374         if (field && is_function_field(field))
2375                 return NULL;
2376
2377         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2378         if (!hist_field)
2379                 return NULL;
2380
2381         hist_field->hist_data = hist_data;
2382
2383         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2384                 goto out; /* caller will populate */
2385
2386         if (flags & HIST_FIELD_FL_VAR_REF) {
2387                 hist_field->fn = hist_field_var_ref;
2388                 goto out;
2389         }
2390
2391         if (flags & HIST_FIELD_FL_HITCOUNT) {
2392                 hist_field->fn = hist_field_counter;
2393                 hist_field->size = sizeof(u64);
2394                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2395                 if (!hist_field->type)
2396                         goto free;
2397                 goto out;
2398         }
2399
2400         if (flags & HIST_FIELD_FL_STACKTRACE) {
2401                 hist_field->fn = hist_field_none;
2402                 goto out;
2403         }
2404
2405         if (flags & HIST_FIELD_FL_LOG2) {
2406                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2407                 hist_field->fn = hist_field_log2;
2408                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2409                 hist_field->size = hist_field->operands[0]->size;
2410                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2411                 if (!hist_field->type)
2412                         goto free;
2413                 goto out;
2414         }
2415
2416         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2417                 hist_field->fn = hist_field_timestamp;
2418                 hist_field->size = sizeof(u64);
2419                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2420                 if (!hist_field->type)
2421                         goto free;
2422                 goto out;
2423         }
2424
2425         if (flags & HIST_FIELD_FL_CPU) {
2426                 hist_field->fn = hist_field_cpu;
2427                 hist_field->size = sizeof(int);
2428                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2429                 if (!hist_field->type)
2430                         goto free;
2431                 goto out;
2432         }
2433
2434         if (WARN_ON_ONCE(!field))
2435                 goto out;
2436
2437         if (is_string_field(field)) {
2438                 flags |= HIST_FIELD_FL_STRING;
2439
2440                 hist_field->size = MAX_FILTER_STR_VAL;
2441                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2442                 if (!hist_field->type)
2443                         goto free;
2444
2445                 if (field->filter_type == FILTER_STATIC_STRING)
2446                         hist_field->fn = hist_field_string;
2447                 else if (field->filter_type == FILTER_DYN_STRING)
2448                         hist_field->fn = hist_field_dynstring;
2449                 else
2450                         hist_field->fn = hist_field_pstring;
2451         } else {
2452                 hist_field->size = field->size;
2453                 hist_field->is_signed = field->is_signed;
2454                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2455                 if (!hist_field->type)
2456                         goto free;
2457
2458                 hist_field->fn = select_value_fn(field->size,
2459                                                  field->is_signed);
2460                 if (!hist_field->fn) {
2461                         destroy_hist_field(hist_field, 0);
2462                         return NULL;
2463                 }
2464         }
2465  out:
2466         hist_field->field = field;
2467         hist_field->flags = flags;
2468
2469         if (var_name) {
2470                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2471                 if (!hist_field->var.name)
2472                         goto free;
2473         }
2474
2475         return hist_field;
2476  free:
2477         destroy_hist_field(hist_field, 0);
2478         return NULL;
2479 }
2480
2481 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2482 {
2483         unsigned int i;
2484
2485         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2486                 if (hist_data->fields[i]) {
2487                         destroy_hist_field(hist_data->fields[i], 0);
2488                         hist_data->fields[i] = NULL;
2489                 }
2490         }
2491
2492         for (i = 0; i < hist_data->n_var_refs; i++) {
2493                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2494                 __destroy_hist_field(hist_data->var_refs[i]);
2495                 hist_data->var_refs[i] = NULL;
2496         }
2497 }
2498
2499 static int init_var_ref(struct hist_field *ref_field,
2500                         struct hist_field *var_field,
2501                         char *system, char *event_name)
2502 {
2503         int err = 0;
2504
2505         ref_field->var.idx = var_field->var.idx;
2506         ref_field->var.hist_data = var_field->hist_data;
2507         ref_field->size = var_field->size;
2508         ref_field->is_signed = var_field->is_signed;
2509         ref_field->flags |= var_field->flags &
2510                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2511
2512         if (system) {
2513                 ref_field->system = kstrdup(system, GFP_KERNEL);
2514                 if (!ref_field->system)
2515                         return -ENOMEM;
2516         }
2517
2518         if (event_name) {
2519                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2520                 if (!ref_field->event_name) {
2521                         err = -ENOMEM;
2522                         goto free;
2523                 }
2524         }
2525
2526         if (var_field->var.name) {
2527                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2528                 if (!ref_field->name) {
2529                         err = -ENOMEM;
2530                         goto free;
2531                 }
2532         } else if (var_field->name) {
2533                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2534                 if (!ref_field->name) {
2535                         err = -ENOMEM;
2536                         goto free;
2537                 }
2538         }
2539
2540         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2541         if (!ref_field->type) {
2542                 err = -ENOMEM;
2543                 goto free;
2544         }
2545  out:
2546         return err;
2547  free:
2548         kfree(ref_field->system);
2549         kfree(ref_field->event_name);
2550         kfree(ref_field->name);
2551
2552         goto out;
2553 }
2554
2555 /**
2556  * create_var_ref - Create a variable reference and attach it to trigger
2557  * @hist_data: The trigger that will be referencing the variable
2558  * @var_field: The VAR field to create a reference to
2559  * @system: The optional system string
2560  * @event_name: The optional event_name string
2561  *
2562  * Given a variable hist_field, create a VAR_REF hist_field that
2563  * represents a reference to it.
2564  *
2565  * This function also adds the reference to the trigger that
2566  * now references the variable.
2567  *
2568  * Return: The VAR_REF field if successful, NULL if not
2569  */
2570 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2571                                          struct hist_field *var_field,
2572                                          char *system, char *event_name)
2573 {
2574         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2575         struct hist_field *ref_field;
2576
2577         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2578         if (ref_field) {
2579                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2580                         destroy_hist_field(ref_field, 0);
2581                         return NULL;
2582                 }
2583
2584                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2585                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2586         }
2587
2588         return ref_field;
2589 }
2590
2591 static bool is_var_ref(char *var_name)
2592 {
2593         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2594                 return false;
2595
2596         return true;
2597 }
2598
2599 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2600                                  char *var_name)
2601 {
2602         char *name, *field;
2603         unsigned int i;
2604
2605         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2606                 name = hist_data->attrs->var_defs.name[i];
2607
2608                 if (strcmp(var_name, name) == 0) {
2609                         field = hist_data->attrs->var_defs.expr[i];
2610                         if (contains_operator(field) || is_var_ref(field))
2611                                 continue;
2612                         return field;
2613                 }
2614         }
2615
2616         return NULL;
2617 }
2618
2619 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2620                                  char *system, char *event_name,
2621                                  char *var_name)
2622 {
2623         struct trace_event_call *call;
2624
2625         if (system && event_name) {
2626                 call = hist_data->event_file->event_call;
2627
2628                 if (strcmp(system, call->class->system) != 0)
2629                         return NULL;
2630
2631                 if (strcmp(event_name, trace_event_name(call)) != 0)
2632                         return NULL;
2633         }
2634
2635         if (!!system != !!event_name)
2636                 return NULL;
2637
2638         if (!is_var_ref(var_name))
2639                 return NULL;
2640
2641         var_name++;
2642
2643         return field_name_from_var(hist_data, var_name);
2644 }
2645
2646 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2647                                         char *system, char *event_name,
2648                                         char *var_name)
2649 {
2650         struct hist_field *var_field = NULL, *ref_field = NULL;
2651
2652         if (!is_var_ref(var_name))
2653                 return NULL;
2654
2655         var_name++;
2656
2657         var_field = find_event_var(hist_data, system, event_name, var_name);
2658         if (var_field)
2659                 ref_field = create_var_ref(hist_data, var_field,
2660                                            system, event_name);
2661
2662         if (!ref_field)
2663                 hist_err_event("Couldn't find variable: $",
2664                                system, event_name, var_name);
2665
2666         return ref_field;
2667 }
2668
2669 static struct ftrace_event_field *
2670 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2671             char *field_str, unsigned long *flags)
2672 {
2673         struct ftrace_event_field *field = NULL;
2674         char *field_name, *modifier, *str;
2675
2676         modifier = str = kstrdup(field_str, GFP_KERNEL);
2677         if (!modifier)
2678                 return ERR_PTR(-ENOMEM);
2679
2680         field_name = strsep(&modifier, ".");
2681         if (modifier) {
2682                 if (strcmp(modifier, "hex") == 0)
2683                         *flags |= HIST_FIELD_FL_HEX;
2684                 else if (strcmp(modifier, "sym") == 0)
2685                         *flags |= HIST_FIELD_FL_SYM;
2686                 else if (strcmp(modifier, "sym-offset") == 0)
2687                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2688                 else if ((strcmp(modifier, "execname") == 0) &&
2689                          (strcmp(field_name, "common_pid") == 0))
2690                         *flags |= HIST_FIELD_FL_EXECNAME;
2691                 else if (strcmp(modifier, "syscall") == 0)
2692                         *flags |= HIST_FIELD_FL_SYSCALL;
2693                 else if (strcmp(modifier, "log2") == 0)
2694                         *flags |= HIST_FIELD_FL_LOG2;
2695                 else if (strcmp(modifier, "usecs") == 0)
2696                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2697                 else {
2698                         hist_err("Invalid field modifier: ", modifier);
2699                         field = ERR_PTR(-EINVAL);
2700                         goto out;
2701                 }
2702         }
2703
2704         if (strcmp(field_name, "common_timestamp") == 0) {
2705                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2706                 hist_data->enable_timestamps = true;
2707                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2708                         hist_data->attrs->ts_in_usecs = true;
2709         } else if (strcmp(field_name, "cpu") == 0)
2710                 *flags |= HIST_FIELD_FL_CPU;
2711         else {
2712                 field = trace_find_event_field(file->event_call, field_name);
2713                 if (!field || !field->size) {
2714                         hist_err("Couldn't find field: ", field_name);
2715                         field = ERR_PTR(-EINVAL);
2716                         goto out;
2717                 }
2718         }
2719  out:
2720         kfree(str);
2721
2722         return field;
2723 }
2724
2725 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2726                                        struct hist_field *var_ref,
2727                                        char *var_name)
2728 {
2729         struct hist_field *alias = NULL;
2730         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2731
2732         alias = create_hist_field(hist_data, NULL, flags, var_name);
2733         if (!alias)
2734                 return NULL;
2735
2736         alias->fn = var_ref->fn;
2737         alias->operands[0] = var_ref;
2738
2739         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2740                 destroy_hist_field(alias, 0);
2741                 return NULL;
2742         }
2743
2744         return alias;
2745 }
2746
2747 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2748                                      struct trace_event_file *file, char *str,
2749                                      unsigned long *flags, char *var_name)
2750 {
2751         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2752         struct ftrace_event_field *field = NULL;
2753         struct hist_field *hist_field = NULL;
2754         int ret = 0;
2755
2756         s = strchr(str, '.');
2757         if (s) {
2758                 s = strchr(++s, '.');
2759                 if (s) {
2760                         ref_system = strsep(&str, ".");
2761                         if (!str) {
2762                                 ret = -EINVAL;
2763                                 goto out;
2764                         }
2765                         ref_event = strsep(&str, ".");
2766                         if (!str) {
2767                                 ret = -EINVAL;
2768                                 goto out;
2769                         }
2770                         ref_var = str;
2771                 }
2772         }
2773
2774         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2775         if (!s) {
2776                 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2777                 if (hist_field) {
2778                         if (var_name) {
2779                                 hist_field = create_alias(hist_data, hist_field, var_name);
2780                                 if (!hist_field) {
2781                                         ret = -ENOMEM;
2782                                         goto out;
2783                                 }
2784                         }
2785                         return hist_field;
2786                 }
2787         } else
2788                 str = s;
2789
2790         field = parse_field(hist_data, file, str, flags);
2791         if (IS_ERR(field)) {
2792                 ret = PTR_ERR(field);
2793                 goto out;
2794         }
2795
2796         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2797         if (!hist_field) {
2798                 ret = -ENOMEM;
2799                 goto out;
2800         }
2801
2802         return hist_field;
2803  out:
2804         return ERR_PTR(ret);
2805 }
2806
2807 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2808                                      struct trace_event_file *file,
2809                                      char *str, unsigned long flags,
2810                                      char *var_name, unsigned int level);
2811
2812 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2813                                       struct trace_event_file *file,
2814                                       char *str, unsigned long flags,
2815                                       char *var_name, unsigned int level)
2816 {
2817         struct hist_field *operand1, *expr = NULL;
2818         unsigned long operand_flags;
2819         int ret = 0;
2820         char *s;
2821
2822         /* we support only -(xxx) i.e. explicit parens required */
2823
2824         if (level > 3) {
2825                 hist_err("Too many subexpressions (3 max): ", str);
2826                 ret = -EINVAL;
2827                 goto free;
2828         }
2829
2830         str++; /* skip leading '-' */
2831
2832         s = strchr(str, '(');
2833         if (s)
2834                 str++;
2835         else {
2836                 ret = -EINVAL;
2837                 goto free;
2838         }
2839
2840         s = strrchr(str, ')');
2841         if (s)
2842                 *s = '\0';
2843         else {
2844                 ret = -EINVAL; /* no closing ')' */
2845                 goto free;
2846         }
2847
2848         flags |= HIST_FIELD_FL_EXPR;
2849         expr = create_hist_field(hist_data, NULL, flags, var_name);
2850         if (!expr) {
2851                 ret = -ENOMEM;
2852                 goto free;
2853         }
2854
2855         operand_flags = 0;
2856         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2857         if (IS_ERR(operand1)) {
2858                 ret = PTR_ERR(operand1);
2859                 goto free;
2860         }
2861
2862         expr->flags |= operand1->flags &
2863                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2864         expr->fn = hist_field_unary_minus;
2865         expr->operands[0] = operand1;
2866         expr->operator = FIELD_OP_UNARY_MINUS;
2867         expr->name = expr_str(expr, 0);
2868         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2869         if (!expr->type) {
2870                 ret = -ENOMEM;
2871                 goto free;
2872         }
2873
2874         return expr;
2875  free:
2876         destroy_hist_field(expr, 0);
2877         return ERR_PTR(ret);
2878 }
2879
2880 static int check_expr_operands(struct hist_field *operand1,
2881                                struct hist_field *operand2)
2882 {
2883         unsigned long operand1_flags = operand1->flags;
2884         unsigned long operand2_flags = operand2->flags;
2885
2886         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2887             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2888                 struct hist_field *var;
2889
2890                 var = find_var_field(operand1->var.hist_data, operand1->name);
2891                 if (!var)
2892                         return -EINVAL;
2893                 operand1_flags = var->flags;
2894         }
2895
2896         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2897             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2898                 struct hist_field *var;
2899
2900                 var = find_var_field(operand2->var.hist_data, operand2->name);
2901                 if (!var)
2902                         return -EINVAL;
2903                 operand2_flags = var->flags;
2904         }
2905
2906         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2907             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2908                 hist_err("Timestamp units in expression don't match", NULL);
2909                 return -EINVAL;
2910         }
2911
2912         return 0;
2913 }
2914
2915 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2916                                      struct trace_event_file *file,
2917                                      char *str, unsigned long flags,
2918                                      char *var_name, unsigned int level)
2919 {
2920         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2921         unsigned long operand_flags;
2922         int field_op, ret = -EINVAL;
2923         char *sep, *operand1_str;
2924
2925         if (level > 3) {
2926                 hist_err("Too many subexpressions (3 max): ", str);
2927                 return ERR_PTR(-EINVAL);
2928         }
2929
2930         field_op = contains_operator(str);
2931
2932         if (field_op == FIELD_OP_NONE)
2933                 return parse_atom(hist_data, file, str, &flags, var_name);
2934
2935         if (field_op == FIELD_OP_UNARY_MINUS)
2936                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2937
2938         switch (field_op) {
2939         case FIELD_OP_MINUS:
2940                 sep = "-";
2941                 break;
2942         case FIELD_OP_PLUS:
2943                 sep = "+";
2944                 break;
2945         default:
2946                 goto free;
2947         }
2948
2949         operand1_str = strsep(&str, sep);
2950         if (!operand1_str || !str)
2951                 goto free;
2952
2953         operand_flags = 0;
2954         operand1 = parse_atom(hist_data, file, operand1_str,
2955                               &operand_flags, NULL);
2956         if (IS_ERR(operand1)) {
2957                 ret = PTR_ERR(operand1);
2958                 operand1 = NULL;
2959                 goto free;
2960         }
2961
2962         /* rest of string could be another expression e.g. b+c in a+b+c */
2963         operand_flags = 0;
2964         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2965         if (IS_ERR(operand2)) {
2966                 ret = PTR_ERR(operand2);
2967                 operand2 = NULL;
2968                 goto free;
2969         }
2970
2971         ret = check_expr_operands(operand1, operand2);
2972         if (ret)
2973                 goto free;
2974
2975         flags |= HIST_FIELD_FL_EXPR;
2976
2977         flags |= operand1->flags &
2978                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2979
2980         expr = create_hist_field(hist_data, NULL, flags, var_name);
2981         if (!expr) {
2982                 ret = -ENOMEM;
2983                 goto free;
2984         }
2985
2986         operand1->read_once = true;
2987         operand2->read_once = true;
2988
2989         expr->operands[0] = operand1;
2990         expr->operands[1] = operand2;
2991         expr->operator = field_op;
2992         expr->name = expr_str(expr, 0);
2993         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2994         if (!expr->type) {
2995                 ret = -ENOMEM;
2996                 goto free;
2997         }
2998
2999         switch (field_op) {
3000         case FIELD_OP_MINUS:
3001                 expr->fn = hist_field_minus;
3002                 break;
3003         case FIELD_OP_PLUS:
3004                 expr->fn = hist_field_plus;
3005                 break;
3006         default:
3007                 ret = -EINVAL;
3008                 goto free;
3009         }
3010
3011         return expr;
3012  free:
3013         destroy_hist_field(operand1, 0);
3014         destroy_hist_field(operand2, 0);
3015         destroy_hist_field(expr, 0);
3016
3017         return ERR_PTR(ret);
3018 }
3019
3020 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3021                                  struct trace_event_file *file)
3022 {
3023         struct event_trigger_data *test;
3024
3025         list_for_each_entry_rcu(test, &file->triggers, list) {
3026                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3027                         if (test->private_data == hist_data)
3028                                 return test->filter_str;
3029                 }
3030         }
3031
3032         return NULL;
3033 }
3034
3035 static struct event_command trigger_hist_cmd;
3036 static int event_hist_trigger_func(struct event_command *cmd_ops,
3037                                    struct trace_event_file *file,
3038                                    char *glob, char *cmd, char *param);
3039
3040 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3041                             struct hist_trigger_data *hist_data,
3042                             unsigned int n_keys)
3043 {
3044         struct hist_field *target_hist_field, *hist_field;
3045         unsigned int n, i, j;
3046
3047         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3048                 return false;
3049
3050         i = hist_data->n_vals;
3051         j = target_hist_data->n_vals;
3052
3053         for (n = 0; n < n_keys; n++) {
3054                 hist_field = hist_data->fields[i + n];
3055                 target_hist_field = target_hist_data->fields[j + n];
3056
3057                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3058                         return false;
3059                 if (hist_field->size != target_hist_field->size)
3060                         return false;
3061                 if (hist_field->is_signed != target_hist_field->is_signed)
3062                         return false;
3063         }
3064
3065         return true;
3066 }
3067
3068 static struct hist_trigger_data *
3069 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3070                      struct trace_event_file *file)
3071 {
3072         struct hist_trigger_data *hist_data;
3073         struct event_trigger_data *test;
3074         unsigned int n_keys;
3075
3076         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3077
3078         list_for_each_entry_rcu(test, &file->triggers, list) {
3079                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3080                         hist_data = test->private_data;
3081
3082                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3083                                 return hist_data;
3084                 }
3085         }
3086
3087         return NULL;
3088 }
3089
3090 static struct trace_event_file *event_file(struct trace_array *tr,
3091                                            char *system, char *event_name)
3092 {
3093         struct trace_event_file *file;
3094
3095         file = __find_event_file(tr, system, event_name);
3096         if (!file)
3097                 return ERR_PTR(-EINVAL);
3098
3099         return file;
3100 }
3101
3102 static struct hist_field *
3103 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3104                          char *system, char *event_name, char *field_name)
3105 {
3106         struct hist_field *event_var;
3107         char *synthetic_name;
3108
3109         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3110         if (!synthetic_name)
3111                 return ERR_PTR(-ENOMEM);
3112
3113         strcpy(synthetic_name, "synthetic_");
3114         strcat(synthetic_name, field_name);
3115
3116         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3117
3118         kfree(synthetic_name);
3119
3120         return event_var;
3121 }
3122
3123 /**
3124  * create_field_var_hist - Automatically create a histogram and var for a field
3125  * @target_hist_data: The target hist trigger
3126  * @subsys_name: Optional subsystem name
3127  * @event_name: Optional event name
3128  * @field_name: The name of the field (and the resulting variable)
3129  *
3130  * Hist trigger actions fetch data from variables, not directly from
3131  * events.  However, for convenience, users are allowed to directly
3132  * specify an event field in an action, which will be automatically
3133  * converted into a variable on their behalf.
3134
3135  * If a user specifies a field on an event that isn't the event the
3136  * histogram currently being defined (the target event histogram), the
3137  * only way that can be accomplished is if a new hist trigger is
3138  * created and the field variable defined on that.
3139  *
3140  * This function creates a new histogram compatible with the target
3141  * event (meaning a histogram with the same key as the target
3142  * histogram), and creates a variable for the specified field, but
3143  * with 'synthetic_' prepended to the variable name in order to avoid
3144  * collision with normal field variables.
3145  *
3146  * Return: The variable created for the field.
3147  */
3148 static struct hist_field *
3149 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3150                       char *subsys_name, char *event_name, char *field_name)
3151 {
3152         struct trace_array *tr = target_hist_data->event_file->tr;
3153         struct hist_field *event_var = ERR_PTR(-EINVAL);
3154         struct hist_trigger_data *hist_data;
3155         unsigned int i, n, first = true;
3156         struct field_var_hist *var_hist;
3157         struct trace_event_file *file;
3158         struct hist_field *key_field;
3159         char *saved_filter;
3160         char *cmd;
3161         int ret;
3162
3163         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3164                 hist_err_event("trace action: Too many field variables defined: ",
3165                                subsys_name, event_name, field_name);
3166                 return ERR_PTR(-EINVAL);
3167         }
3168
3169         file = event_file(tr, subsys_name, event_name);
3170
3171         if (IS_ERR(file)) {
3172                 hist_err_event("trace action: Event file not found: ",
3173                                subsys_name, event_name, field_name);
3174                 ret = PTR_ERR(file);
3175                 return ERR_PTR(ret);
3176         }
3177
3178         /*
3179          * Look for a histogram compatible with target.  We'll use the
3180          * found histogram specification to create a new matching
3181          * histogram with our variable on it.  target_hist_data is not
3182          * yet a registered histogram so we can't use that.
3183          */
3184         hist_data = find_compatible_hist(target_hist_data, file);
3185         if (!hist_data) {
3186                 hist_err_event("trace action: Matching event histogram not found: ",
3187                                subsys_name, event_name, field_name);
3188                 return ERR_PTR(-EINVAL);
3189         }
3190
3191         /* See if a synthetic field variable has already been created */
3192         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3193                                              event_name, field_name);
3194         if (!IS_ERR_OR_NULL(event_var))
3195                 return event_var;
3196
3197         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3198         if (!var_hist)
3199                 return ERR_PTR(-ENOMEM);
3200
3201         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3202         if (!cmd) {
3203                 kfree(var_hist);
3204                 return ERR_PTR(-ENOMEM);
3205         }
3206
3207         /* Use the same keys as the compatible histogram */
3208         strcat(cmd, "keys=");
3209
3210         for_each_hist_key_field(i, hist_data) {
3211                 key_field = hist_data->fields[i];
3212                 if (!first)
3213                         strcat(cmd, ",");
3214                 strcat(cmd, key_field->field->name);
3215                 first = false;
3216         }
3217
3218         /* Create the synthetic field variable specification */
3219         strcat(cmd, ":synthetic_");
3220         strcat(cmd, field_name);
3221         strcat(cmd, "=");
3222         strcat(cmd, field_name);
3223
3224         /* Use the same filter as the compatible histogram */
3225         saved_filter = find_trigger_filter(hist_data, file);
3226         if (saved_filter) {
3227                 strcat(cmd, " if ");
3228                 strcat(cmd, saved_filter);
3229         }
3230
3231         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3232         if (!var_hist->cmd) {
3233                 kfree(cmd);
3234                 kfree(var_hist);
3235                 return ERR_PTR(-ENOMEM);
3236         }
3237
3238         /* Save the compatible histogram information */
3239         var_hist->hist_data = hist_data;
3240
3241         /* Create the new histogram with our variable */
3242         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3243                                       "", "hist", cmd);
3244         if (ret) {
3245                 kfree(cmd);
3246                 kfree(var_hist->cmd);
3247                 kfree(var_hist);
3248                 hist_err_event("trace action: Couldn't create histogram for field: ",
3249                                subsys_name, event_name, field_name);
3250                 return ERR_PTR(ret);
3251         }
3252
3253         kfree(cmd);
3254
3255         /* If we can't find the variable, something went wrong */
3256         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3257                                              event_name, field_name);
3258         if (IS_ERR_OR_NULL(event_var)) {
3259                 kfree(var_hist->cmd);
3260                 kfree(var_hist);
3261                 hist_err_event("trace action: Couldn't find synthetic variable: ",
3262                                subsys_name, event_name, field_name);
3263                 return ERR_PTR(-EINVAL);
3264         }
3265
3266         n = target_hist_data->n_field_var_hists;
3267         target_hist_data->field_var_hists[n] = var_hist;
3268         target_hist_data->n_field_var_hists++;
3269
3270         return event_var;
3271 }
3272
3273 static struct hist_field *
3274 find_target_event_var(struct hist_trigger_data *hist_data,
3275                       char *subsys_name, char *event_name, char *var_name)
3276 {
3277         struct trace_event_file *file = hist_data->event_file;
3278         struct hist_field *hist_field = NULL;
3279
3280         if (subsys_name) {
3281                 struct trace_event_call *call;
3282
3283                 if (!event_name)
3284                         return NULL;
3285
3286                 call = file->event_call;
3287
3288                 if (strcmp(subsys_name, call->class->system) != 0)
3289                         return NULL;
3290
3291                 if (strcmp(event_name, trace_event_name(call)) != 0)
3292                         return NULL;
3293         }
3294
3295         hist_field = find_var_field(hist_data, var_name);
3296
3297         return hist_field;
3298 }
3299
3300 static inline void __update_field_vars(struct tracing_map_elt *elt,
3301                                        struct ring_buffer_event *rbe,
3302                                        void *rec,
3303                                        struct field_var **field_vars,
3304                                        unsigned int n_field_vars,
3305                                        unsigned int field_var_str_start)
3306 {
3307         struct hist_elt_data *elt_data = elt->private_data;
3308         unsigned int i, j, var_idx;
3309         u64 var_val;
3310
3311         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3312                 struct field_var *field_var = field_vars[i];
3313                 struct hist_field *var = field_var->var;
3314                 struct hist_field *val = field_var->val;
3315
3316                 var_val = val->fn(val, elt, rbe, rec);
3317                 var_idx = var->var.idx;
3318
3319                 if (val->flags & HIST_FIELD_FL_STRING) {
3320                         char *str = elt_data->field_var_str[j++];
3321                         char *val_str = (char *)(uintptr_t)var_val;
3322
3323                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3324                         var_val = (u64)(uintptr_t)str;
3325                 }
3326                 tracing_map_set_var(elt, var_idx, var_val);
3327         }
3328 }
3329
3330 static void update_field_vars(struct hist_trigger_data *hist_data,
3331                               struct tracing_map_elt *elt,
3332                               struct ring_buffer_event *rbe,
3333                               void *rec)
3334 {
3335         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3336                             hist_data->n_field_vars, 0);
3337 }
3338
3339 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3340                                  struct tracing_map_elt *elt, void *rec,
3341                                  struct ring_buffer_event *rbe, void *key,
3342                                  struct action_data *data, u64 *var_ref_vals)
3343 {
3344         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
3345                             hist_data->n_save_vars, hist_data->n_field_var_str);
3346 }
3347
3348 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3349                                      struct trace_event_file *file,
3350                                      char *name, int size, const char *type)
3351 {
3352         struct hist_field *var;
3353         int idx;
3354
3355         if (find_var(hist_data, file, name) && !hist_data->remove) {
3356                 var = ERR_PTR(-EINVAL);
3357                 goto out;
3358         }
3359
3360         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3361         if (!var) {
3362                 var = ERR_PTR(-ENOMEM);
3363                 goto out;
3364         }
3365
3366         idx = tracing_map_add_var(hist_data->map);
3367         if (idx < 0) {
3368                 kfree(var);
3369                 var = ERR_PTR(-EINVAL);
3370                 goto out;
3371         }
3372
3373         var->flags = HIST_FIELD_FL_VAR;
3374         var->var.idx = idx;
3375         var->var.hist_data = var->hist_data = hist_data;
3376         var->size = size;
3377         var->var.name = kstrdup(name, GFP_KERNEL);
3378         var->type = kstrdup(type, GFP_KERNEL);
3379         if (!var->var.name || !var->type) {
3380                 kfree(var->var.name);
3381                 kfree(var->type);
3382                 kfree(var);
3383                 var = ERR_PTR(-ENOMEM);
3384         }
3385  out:
3386         return var;
3387 }
3388
3389 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3390                                           struct trace_event_file *file,
3391                                           char *field_name)
3392 {
3393         struct hist_field *val = NULL, *var = NULL;
3394         unsigned long flags = HIST_FIELD_FL_VAR;
3395         struct field_var *field_var;
3396         int ret = 0;
3397
3398         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3399                 hist_err("Too many field variables defined: ", field_name);
3400                 ret = -EINVAL;
3401                 goto err;
3402         }
3403
3404         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3405         if (IS_ERR(val)) {
3406                 hist_err("Couldn't parse field variable: ", field_name);
3407                 ret = PTR_ERR(val);
3408                 goto err;
3409         }
3410
3411         var = create_var(hist_data, file, field_name, val->size, val->type);
3412         if (IS_ERR(var)) {
3413                 hist_err("Couldn't create or find variable: ", field_name);
3414                 kfree(val);
3415                 ret = PTR_ERR(var);
3416                 goto err;
3417         }
3418
3419         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3420         if (!field_var) {
3421                 kfree(val);
3422                 kfree(var);
3423                 ret =  -ENOMEM;
3424                 goto err;
3425         }
3426
3427         field_var->var = var;
3428         field_var->val = val;
3429  out:
3430         return field_var;
3431  err:
3432         field_var = ERR_PTR(ret);
3433         goto out;
3434 }
3435
3436 /**
3437  * create_target_field_var - Automatically create a variable for a field
3438  * @target_hist_data: The target hist trigger
3439  * @subsys_name: Optional subsystem name
3440  * @event_name: Optional event name
3441  * @var_name: The name of the field (and the resulting variable)
3442  *
3443  * Hist trigger actions fetch data from variables, not directly from
3444  * events.  However, for convenience, users are allowed to directly
3445  * specify an event field in an action, which will be automatically
3446  * converted into a variable on their behalf.
3447
3448  * This function creates a field variable with the name var_name on
3449  * the hist trigger currently being defined on the target event.  If
3450  * subsys_name and event_name are specified, this function simply
3451  * verifies that they do in fact match the target event subsystem and
3452  * event name.
3453  *
3454  * Return: The variable created for the field.
3455  */
3456 static struct field_var *
3457 create_target_field_var(struct hist_trigger_data *target_hist_data,
3458                         char *subsys_name, char *event_name, char *var_name)
3459 {
3460         struct trace_event_file *file = target_hist_data->event_file;
3461
3462         if (subsys_name) {
3463                 struct trace_event_call *call;
3464
3465                 if (!event_name)
3466                         return NULL;
3467
3468                 call = file->event_call;
3469
3470                 if (strcmp(subsys_name, call->class->system) != 0)
3471                         return NULL;
3472
3473                 if (strcmp(event_name, trace_event_name(call)) != 0)
3474                         return NULL;
3475         }
3476
3477         return create_field_var(target_hist_data, file, var_name);
3478 }
3479
3480 static bool check_track_val_max(u64 track_val, u64 var_val)
3481 {
3482         if (var_val <= track_val)
3483                 return false;
3484
3485         return true;
3486 }
3487
3488 static bool check_track_val_changed(u64 track_val, u64 var_val)
3489 {
3490         if (var_val == track_val)
3491                 return false;
3492
3493         return true;
3494 }
3495
3496 static u64 get_track_val(struct hist_trigger_data *hist_data,
3497                          struct tracing_map_elt *elt,
3498                          struct action_data *data)
3499 {
3500         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3501         u64 track_val;
3502
3503         track_val = tracing_map_read_var(elt, track_var_idx);
3504
3505         return track_val;
3506 }
3507
3508 static void save_track_val(struct hist_trigger_data *hist_data,
3509                            struct tracing_map_elt *elt,
3510                            struct action_data *data, u64 var_val)
3511 {
3512         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3513
3514         tracing_map_set_var(elt, track_var_idx, var_val);
3515 }
3516
3517 static void save_track_data(struct hist_trigger_data *hist_data,
3518                             struct tracing_map_elt *elt, void *rec,
3519                             struct ring_buffer_event *rbe, void *key,
3520                             struct action_data *data, u64 *var_ref_vals)
3521 {
3522         if (data->track_data.save_data)
3523                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3524 }
3525
3526 static bool check_track_val(struct tracing_map_elt *elt,
3527                             struct action_data *data,
3528                             u64 var_val)
3529 {
3530         struct hist_trigger_data *hist_data;
3531         u64 track_val;
3532
3533         hist_data = data->track_data.track_var->hist_data;
3534         track_val = get_track_val(hist_data, elt, data);
3535
3536         return data->track_data.check_val(track_val, var_val);
3537 }
3538
3539 #ifdef CONFIG_TRACER_SNAPSHOT
3540 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3541 {
3542         /* called with tr->max_lock held */
3543         struct track_data *track_data = tr->cond_snapshot->cond_data;
3544         struct hist_elt_data *elt_data, *track_elt_data;
3545         struct snapshot_context *context = cond_data;
3546         u64 track_val;
3547
3548         if (!track_data)
3549                 return false;
3550
3551         track_val = get_track_val(track_data->hist_data, context->elt,
3552                                   track_data->action_data);
3553
3554         track_data->track_val = track_val;
3555         memcpy(track_data->key, context->key, track_data->key_len);
3556
3557         elt_data = context->elt->private_data;
3558         track_elt_data = track_data->elt.private_data;
3559         if (elt_data->comm)
3560                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3561
3562         track_data->updated = true;
3563
3564         return true;
3565 }
3566
3567 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3568                                      struct tracing_map_elt *elt, void *rec,
3569                                      struct ring_buffer_event *rbe, void *key,
3570                                      struct action_data *data,
3571                                      u64 *var_ref_vals)
3572 {
3573         struct trace_event_file *file = hist_data->event_file;
3574         struct snapshot_context context;
3575
3576         context.elt = elt;
3577         context.key = key;
3578
3579         tracing_snapshot_cond(file->tr, &context);
3580 }
3581
3582 static void hist_trigger_print_key(struct seq_file *m,
3583                                    struct hist_trigger_data *hist_data,
3584                                    void *key,
3585                                    struct tracing_map_elt *elt);
3586
3587 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3588 {
3589         unsigned int i;
3590
3591         if (!hist_data->n_actions)
3592                 return NULL;
3593
3594         for (i = 0; i < hist_data->n_actions; i++) {
3595                 struct action_data *data = hist_data->actions[i];
3596
3597                 if (data->action == ACTION_SNAPSHOT)
3598                         return data;
3599         }
3600
3601         return NULL;
3602 }
3603
3604 static void track_data_snapshot_print(struct seq_file *m,
3605                                       struct hist_trigger_data *hist_data)
3606 {
3607         struct trace_event_file *file = hist_data->event_file;
3608         struct track_data *track_data;
3609         struct action_data *action;
3610
3611         track_data = tracing_cond_snapshot_data(file->tr);
3612         if (!track_data)
3613                 return;
3614
3615         if (!track_data->updated)
3616                 return;
3617
3618         action = snapshot_action(hist_data);
3619         if (!action)
3620                 return;
3621
3622         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3623         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3624                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3625                    action->track_data.var_str, track_data->track_val);
3626
3627         seq_puts(m, "\ttriggered by event with key: ");
3628         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3629         seq_putc(m, '\n');
3630 }
3631 #else
3632 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3633 {
3634         return false;
3635 }
3636 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3637                                      struct tracing_map_elt *elt, void *rec,
3638                                      struct ring_buffer_event *rbe, void *key,
3639                                      struct action_data *data,
3640                                      u64 *var_ref_vals) {}
3641 static void track_data_snapshot_print(struct seq_file *m,
3642                                       struct hist_trigger_data *hist_data) {}
3643 #endif /* CONFIG_TRACER_SNAPSHOT */
3644
3645 static void track_data_print(struct seq_file *m,
3646                              struct hist_trigger_data *hist_data,
3647                              struct tracing_map_elt *elt,
3648                              struct action_data *data)
3649 {
3650         u64 track_val = get_track_val(hist_data, elt, data);
3651         unsigned int i, save_var_idx;
3652
3653         if (data->handler == HANDLER_ONMAX)
3654                 seq_printf(m, "\n\tmax: %10llu", track_val);
3655         else if (data->handler == HANDLER_ONCHANGE)
3656                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3657
3658         if (data->action == ACTION_SNAPSHOT)
3659                 return;
3660
3661         for (i = 0; i < hist_data->n_save_vars; i++) {
3662                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3663                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3664                 u64 val;
3665
3666                 save_var_idx = save_var->var.idx;
3667
3668                 val = tracing_map_read_var(elt, save_var_idx);
3669
3670                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3671                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3672                                    (char *)(uintptr_t)(val));
3673                 } else
3674                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3675         }
3676 }
3677
3678 static void ontrack_action(struct hist_trigger_data *hist_data,
3679                            struct tracing_map_elt *elt, void *rec,
3680                            struct ring_buffer_event *rbe, void *key,
3681                            struct action_data *data, u64 *var_ref_vals)
3682 {
3683         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3684
3685         if (check_track_val(elt, data, var_val)) {
3686                 save_track_val(hist_data, elt, data, var_val);
3687                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3688         }
3689 }
3690
3691 static void action_data_destroy(struct action_data *data)
3692 {
3693         unsigned int i;
3694
3695         lockdep_assert_held(&event_mutex);
3696
3697         kfree(data->action_name);
3698
3699         for (i = 0; i < data->n_params; i++)
3700                 kfree(data->params[i]);
3701
3702         if (data->synth_event)
3703                 data->synth_event->ref--;
3704
3705         kfree(data->synth_event_name);
3706
3707         kfree(data);
3708 }
3709
3710 static void track_data_destroy(struct hist_trigger_data *hist_data,
3711                                struct action_data *data)
3712 {
3713         struct trace_event_file *file = hist_data->event_file;
3714
3715         destroy_hist_field(data->track_data.track_var, 0);
3716         destroy_hist_field(data->track_data.var_ref, 0);
3717
3718         if (data->action == ACTION_SNAPSHOT) {
3719                 struct track_data *track_data;
3720
3721                 track_data = tracing_cond_snapshot_data(file->tr);
3722                 if (track_data && track_data->hist_data == hist_data) {
3723                         tracing_snapshot_cond_disable(file->tr);
3724                         track_data_free(track_data);
3725                 }
3726         }
3727
3728         kfree(data->track_data.var_str);
3729
3730         action_data_destroy(data);
3731 }
3732
3733 static int action_create(struct hist_trigger_data *hist_data,
3734                          struct action_data *data);
3735
3736 static int track_data_create(struct hist_trigger_data *hist_data,
3737                              struct action_data *data)
3738 {
3739         struct hist_field *var_field, *ref_field, *track_var = NULL;
3740         struct trace_event_file *file = hist_data->event_file;
3741         char *track_data_var_str;
3742         int ret = 0;
3743
3744         track_data_var_str = data->track_data.var_str;
3745         if (track_data_var_str[0] != '$') {
3746                 hist_err("For onmax(x) or onchange(x), x must be a variable: ", track_data_var_str);
3747                 return -EINVAL;
3748         }
3749         track_data_var_str++;
3750
3751         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3752         if (!var_field) {
3753                 hist_err("Couldn't find onmax or onchange variable: ", track_data_var_str);
3754                 return -EINVAL;
3755         }
3756
3757         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3758         if (!ref_field)
3759                 return -ENOMEM;
3760
3761         data->track_data.var_ref = ref_field;
3762
3763         if (data->handler == HANDLER_ONMAX)
3764                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3765         if (IS_ERR(track_var)) {
3766                 hist_err("Couldn't create onmax variable: ", "__max");
3767                 ret = PTR_ERR(track_var);
3768                 goto out;
3769         }
3770
3771         if (data->handler == HANDLER_ONCHANGE)
3772                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3773         if (IS_ERR(track_var)) {
3774                 hist_err("Couldn't create onchange variable: ", "__change");
3775                 ret = PTR_ERR(track_var);
3776                 goto out;
3777         }
3778         data->track_data.track_var = track_var;
3779
3780         ret = action_create(hist_data, data);
3781  out:
3782         return ret;
3783 }
3784
3785 static int parse_action_params(char *params, struct action_data *data)
3786 {
3787         char *param, *saved_param;
3788         bool first_param = true;
3789         int ret = 0;
3790
3791         while (params) {
3792                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3793                         hist_err("Too many action params", "");
3794                         goto out;
3795                 }
3796
3797                 param = strsep(&params, ",");
3798                 if (!param) {
3799                         hist_err("No action param found", "");
3800                         ret = -EINVAL;
3801                         goto out;
3802                 }
3803
3804                 param = strstrip(param);
3805                 if (strlen(param) < 2) {
3806                         hist_err("Invalid action param: ", param);
3807                         ret = -EINVAL;
3808                         goto out;
3809                 }
3810
3811                 saved_param = kstrdup(param, GFP_KERNEL);
3812                 if (!saved_param) {
3813                         ret = -ENOMEM;
3814                         goto out;
3815                 }
3816
3817                 if (first_param && data->use_trace_keyword) {
3818                         data->synth_event_name = saved_param;
3819                         first_param = false;
3820                         continue;
3821                 }
3822                 first_param = false;
3823
3824                 data->params[data->n_params++] = saved_param;
3825         }
3826  out:
3827         return ret;
3828 }
3829
3830 static int action_parse(char *str, struct action_data *data,
3831                         enum handler_id handler)
3832 {
3833         char *action_name;
3834         int ret = 0;
3835
3836         strsep(&str, ".");
3837         if (!str) {
3838                 hist_err("action parsing: No action found", "");
3839                 ret = -EINVAL;
3840                 goto out;
3841         }
3842
3843         action_name = strsep(&str, "(");
3844         if (!action_name || !str) {
3845                 hist_err("action parsing: No action found", "");
3846                 ret = -EINVAL;
3847                 goto out;
3848         }
3849
3850         if (str_has_prefix(action_name, "save")) {
3851                 char *params = strsep(&str, ")");
3852
3853                 if (!params) {
3854                         hist_err("action parsing: No params found for %s", "save");
3855                         ret = -EINVAL;
3856                         goto out;
3857                 }
3858
3859                 ret = parse_action_params(params, data);
3860                 if (ret)
3861                         goto out;
3862
3863                 if (handler == HANDLER_ONMAX)
3864                         data->track_data.check_val = check_track_val_max;
3865                 else if (handler == HANDLER_ONCHANGE)
3866                         data->track_data.check_val = check_track_val_changed;
3867                 else {
3868                         hist_err("action parsing: Handler doesn't support action: ", action_name);
3869                         ret = -EINVAL;
3870                         goto out;
3871                 }
3872
3873                 data->track_data.save_data = save_track_data_vars;
3874                 data->fn = ontrack_action;
3875                 data->action = ACTION_SAVE;
3876         } else if (str_has_prefix(action_name, "snapshot")) {
3877                 char *params = strsep(&str, ")");
3878
3879                 if (!str) {
3880                         hist_err("action parsing: No closing paren found: %s", params);
3881                         ret = -EINVAL;
3882                         goto out;
3883                 }
3884
3885                 if (handler == HANDLER_ONMAX)
3886                         data->track_data.check_val = check_track_val_max;
3887                 else if (handler == HANDLER_ONCHANGE)
3888                         data->track_data.check_val = check_track_val_changed;
3889                 else {
3890                         hist_err("action parsing: Handler doesn't support action: ", action_name);
3891                         ret = -EINVAL;
3892                         goto out;
3893                 }
3894
3895                 data->track_data.save_data = save_track_data_snapshot;
3896                 data->fn = ontrack_action;
3897                 data->action = ACTION_SNAPSHOT;
3898         } else {
3899                 char *params = strsep(&str, ")");
3900
3901                 if (str_has_prefix(action_name, "trace"))
3902                         data->use_trace_keyword = true;
3903
3904                 if (params) {
3905                         ret = parse_action_params(params, data);
3906                         if (ret)
3907                                 goto out;
3908                 }
3909
3910                 if (handler == HANDLER_ONMAX)
3911                         data->track_data.check_val = check_track_val_max;
3912                 else if (handler == HANDLER_ONCHANGE)
3913                         data->track_data.check_val = check_track_val_changed;
3914
3915                 if (handler != HANDLER_ONMATCH) {
3916                         data->track_data.save_data = action_trace;
3917                         data->fn = ontrack_action;
3918                 } else
3919                         data->fn = action_trace;
3920
3921                 data->action = ACTION_TRACE;
3922         }
3923
3924         data->action_name = kstrdup(action_name, GFP_KERNEL);
3925         if (!data->action_name) {
3926                 ret = -ENOMEM;
3927                 goto out;
3928         }
3929
3930         data->handler = handler;
3931  out:
3932         return ret;
3933 }
3934
3935 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3936                                             char *str, enum handler_id handler)
3937 {
3938         struct action_data *data;
3939         int ret = -EINVAL;
3940         char *var_str;
3941
3942         data = kzalloc(sizeof(*data), GFP_KERNEL);
3943         if (!data)
3944                 return ERR_PTR(-ENOMEM);
3945
3946         var_str = strsep(&str, ")");
3947         if (!var_str || !str) {
3948                 ret = -EINVAL;
3949                 goto free;
3950         }
3951
3952         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3953         if (!data->track_data.var_str) {
3954                 ret = -ENOMEM;
3955                 goto free;
3956         }
3957
3958         ret = action_parse(str, data, handler);
3959         if (ret)
3960                 goto free;
3961  out:
3962         return data;
3963  free:
3964         track_data_destroy(hist_data, data);
3965         data = ERR_PTR(ret);
3966         goto out;
3967 }
3968
3969 static void onmatch_destroy(struct action_data *data)
3970 {
3971         kfree(data->match_data.event);
3972         kfree(data->match_data.event_system);
3973
3974         action_data_destroy(data);
3975 }
3976
3977 static void destroy_field_var(struct field_var *field_var)
3978 {
3979         if (!field_var)
3980                 return;
3981
3982         destroy_hist_field(field_var->var, 0);
3983         destroy_hist_field(field_var->val, 0);
3984
3985         kfree(field_var);
3986 }
3987
3988 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3989 {
3990         unsigned int i;
3991
3992         for (i = 0; i < hist_data->n_field_vars; i++)
3993                 destroy_field_var(hist_data->field_vars[i]);
3994 }
3995
3996 static void save_field_var(struct hist_trigger_data *hist_data,
3997                            struct field_var *field_var)
3998 {
3999         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
4000
4001         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4002                 hist_data->n_field_var_str++;
4003 }
4004
4005
4006 static int check_synth_field(struct synth_event *event,
4007                              struct hist_field *hist_field,
4008                              unsigned int field_pos)
4009 {
4010         struct synth_field *field;
4011
4012         if (field_pos >= event->n_fields)
4013                 return -EINVAL;
4014
4015         field = event->fields[field_pos];
4016
4017         if (strcmp(field->type, hist_field->type) != 0)
4018                 return -EINVAL;
4019
4020         return 0;
4021 }
4022
4023 static struct hist_field *
4024 trace_action_find_var(struct hist_trigger_data *hist_data,
4025                       struct action_data *data,
4026                       char *system, char *event, char *var)
4027 {
4028         struct hist_field *hist_field;
4029
4030         var++; /* skip '$' */
4031
4032         hist_field = find_target_event_var(hist_data, system, event, var);
4033         if (!hist_field) {
4034                 if (!system && data->handler == HANDLER_ONMATCH) {
4035                         system = data->match_data.event_system;
4036                         event = data->match_data.event;
4037                 }
4038
4039                 hist_field = find_event_var(hist_data, system, event, var);
4040         }
4041
4042         if (!hist_field)
4043                 hist_err_event("trace action: Couldn't find param: $", system, event, var);
4044
4045         return hist_field;
4046 }
4047
4048 static struct hist_field *
4049 trace_action_create_field_var(struct hist_trigger_data *hist_data,
4050                               struct action_data *data, char *system,
4051                               char *event, char *var)
4052 {
4053         struct hist_field *hist_field = NULL;
4054         struct field_var *field_var;
4055
4056         /*
4057          * First try to create a field var on the target event (the
4058          * currently being defined).  This will create a variable for
4059          * unqualified fields on the target event, or if qualified,
4060          * target fields that have qualified names matching the target.
4061          */
4062         field_var = create_target_field_var(hist_data, system, event, var);
4063
4064         if (field_var && !IS_ERR(field_var)) {
4065                 save_field_var(hist_data, field_var);
4066                 hist_field = field_var->var;
4067         } else {
4068                 field_var = NULL;
4069                 /*
4070                  * If no explicit system.event is specfied, default to
4071                  * looking for fields on the onmatch(system.event.xxx)
4072                  * event.
4073                  */
4074                 if (!system && data->handler == HANDLER_ONMATCH) {
4075                         system = data->match_data.event_system;
4076                         event = data->match_data.event;
4077                 }
4078
4079                 /*
4080                  * At this point, we're looking at a field on another
4081                  * event.  Because we can't modify a hist trigger on
4082                  * another event to add a variable for a field, we need
4083                  * to create a new trigger on that event and create the
4084                  * variable at the same time.
4085                  */
4086                 hist_field = create_field_var_hist(hist_data, system, event, var);
4087                 if (IS_ERR(hist_field))
4088                         goto free;
4089         }
4090  out:
4091         return hist_field;
4092  free:
4093         destroy_field_var(field_var);
4094         hist_field = NULL;
4095         goto out;
4096 }
4097
4098 static int trace_action_create(struct hist_trigger_data *hist_data,
4099                                struct action_data *data)
4100 {
4101         char *event_name, *param, *system = NULL;
4102         struct hist_field *hist_field, *var_ref;
4103         unsigned int i, var_ref_idx;
4104         unsigned int field_pos = 0;
4105         struct synth_event *event;
4106         char *synth_event_name;
4107         int ret = 0;
4108
4109         lockdep_assert_held(&event_mutex);
4110
4111         if (data->use_trace_keyword)
4112                 synth_event_name = data->synth_event_name;
4113         else
4114                 synth_event_name = data->action_name;
4115
4116         event = find_synth_event(synth_event_name);
4117         if (!event) {
4118                 hist_err("trace action: Couldn't find synthetic event: ", synth_event_name);
4119                 return -EINVAL;
4120         }
4121
4122         event->ref++;
4123
4124         var_ref_idx = hist_data->n_var_refs;
4125
4126         for (i = 0; i < data->n_params; i++) {
4127                 char *p;
4128
4129                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4130                 if (!param) {
4131                         ret = -ENOMEM;
4132                         goto err;
4133                 }
4134
4135                 system = strsep(&param, ".");
4136                 if (!param) {
4137                         param = (char *)system;
4138                         system = event_name = NULL;
4139                 } else {
4140                         event_name = strsep(&param, ".");
4141                         if (!param) {
4142                                 kfree(p);
4143                                 ret = -EINVAL;
4144                                 goto err;
4145                         }
4146                 }
4147
4148                 if (param[0] == '$')
4149                         hist_field = trace_action_find_var(hist_data, data,
4150                                                            system, event_name,
4151                                                            param);
4152                 else
4153                         hist_field = trace_action_create_field_var(hist_data,
4154                                                                    data,
4155                                                                    system,
4156                                                                    event_name,
4157                                                                    param);
4158
4159                 if (!hist_field) {
4160                         kfree(p);
4161                         ret = -EINVAL;
4162                         goto err;
4163                 }
4164
4165                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4166                         var_ref = create_var_ref(hist_data, hist_field,
4167                                                  system, event_name);
4168                         if (!var_ref) {
4169                                 kfree(p);
4170                                 ret = -ENOMEM;
4171                                 goto err;
4172                         }
4173
4174                         field_pos++;
4175                         kfree(p);
4176                         continue;
4177                 }
4178
4179                 hist_err_event("trace action: Param type doesn't match synthetic event field type: ",
4180                                system, event_name, param);
4181                 kfree(p);
4182                 ret = -EINVAL;
4183                 goto err;
4184         }
4185
4186         if (field_pos != event->n_fields) {
4187                 hist_err("trace action: Param count doesn't match synthetic event field count: ", event->name);
4188                 ret = -EINVAL;
4189                 goto err;
4190         }
4191
4192         data->synth_event = event;
4193         data->var_ref_idx = var_ref_idx;
4194  out:
4195         return ret;
4196  err:
4197         event->ref--;
4198
4199         goto out;
4200 }
4201
4202 static int action_create(struct hist_trigger_data *hist_data,
4203                          struct action_data *data)
4204 {
4205         struct trace_event_file *file = hist_data->event_file;
4206         struct track_data *track_data;
4207         struct field_var *field_var;
4208         unsigned int i;
4209         char *param;
4210         int ret = 0;
4211
4212         if (data->action == ACTION_TRACE)
4213                 return trace_action_create(hist_data, data);
4214
4215         if (data->action == ACTION_SNAPSHOT) {
4216                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4217                 if (IS_ERR(track_data)) {
4218                         ret = PTR_ERR(track_data);
4219                         goto out;
4220                 }
4221
4222                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4223                                                    cond_snapshot_update);
4224                 if (ret)
4225                         track_data_free(track_data);
4226
4227                 goto out;
4228         }
4229
4230         if (data->action == ACTION_SAVE) {
4231                 if (hist_data->n_save_vars) {
4232                         ret = -EEXIST;
4233                         hist_err("save action: Can't have more than one save() action per hist", "");
4234                         goto out;
4235                 }
4236
4237                 for (i = 0; i < data->n_params; i++) {
4238                         param = kstrdup(data->params[i], GFP_KERNEL);
4239                         if (!param) {
4240                                 ret = -ENOMEM;
4241                                 goto out;
4242                         }
4243
4244                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4245                         if (IS_ERR(field_var)) {
4246                                 hist_err("save action: Couldn't create field variable: ", param);
4247                                 ret = PTR_ERR(field_var);
4248                                 kfree(param);
4249                                 goto out;
4250                         }
4251
4252                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4253                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4254                                 hist_data->n_save_var_str++;
4255                         kfree(param);
4256                 }
4257         }
4258  out:
4259         return ret;
4260 }
4261
4262 static int onmatch_create(struct hist_trigger_data *hist_data,
4263                           struct action_data *data)
4264 {
4265         return action_create(hist_data, data);
4266 }
4267
4268 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4269 {
4270         char *match_event, *match_event_system;
4271         struct action_data *data;
4272         int ret = -EINVAL;
4273
4274         data = kzalloc(sizeof(*data), GFP_KERNEL);
4275         if (!data)
4276                 return ERR_PTR(-ENOMEM);
4277
4278         match_event = strsep(&str, ")");
4279         if (!match_event || !str) {
4280                 hist_err("onmatch: Missing closing paren: ", match_event);
4281                 goto free;
4282         }
4283
4284         match_event_system = strsep(&match_event, ".");
4285         if (!match_event) {
4286                 hist_err("onmatch: Missing subsystem for match event: ", match_event_system);
4287                 goto free;
4288         }
4289
4290         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4291                 hist_err_event("onmatch: Invalid subsystem or event name: ",
4292                                match_event_system, match_event, NULL);
4293                 goto free;
4294         }
4295
4296         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4297         if (!data->match_data.event) {
4298                 ret = -ENOMEM;
4299                 goto free;
4300         }
4301
4302         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4303         if (!data->match_data.event_system) {
4304                 ret = -ENOMEM;
4305                 goto free;
4306         }
4307
4308         ret = action_parse(str, data, HANDLER_ONMATCH);
4309         if (ret)
4310                 goto free;
4311  out:
4312         return data;
4313  free:
4314         onmatch_destroy(data);
4315         data = ERR_PTR(ret);
4316         goto out;
4317 }
4318
4319 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4320 {
4321         hist_data->fields[HITCOUNT_IDX] =
4322                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4323         if (!hist_data->fields[HITCOUNT_IDX])
4324                 return -ENOMEM;
4325
4326         hist_data->n_vals++;
4327         hist_data->n_fields++;
4328
4329         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4330                 return -EINVAL;
4331
4332         return 0;
4333 }
4334
4335 static int __create_val_field(struct hist_trigger_data *hist_data,
4336                               unsigned int val_idx,
4337                               struct trace_event_file *file,
4338                               char *var_name, char *field_str,
4339                               unsigned long flags)
4340 {
4341         struct hist_field *hist_field;
4342         int ret = 0;
4343
4344         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4345         if (IS_ERR(hist_field)) {
4346                 ret = PTR_ERR(hist_field);
4347                 goto out;
4348         }
4349
4350         hist_data->fields[val_idx] = hist_field;
4351
4352         ++hist_data->n_vals;
4353         ++hist_data->n_fields;
4354
4355         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4356                 ret = -EINVAL;
4357  out:
4358         return ret;
4359 }
4360
4361 static int create_val_field(struct hist_trigger_data *hist_data,
4362                             unsigned int val_idx,
4363                             struct trace_event_file *file,
4364                             char *field_str)
4365 {
4366         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4367                 return -EINVAL;
4368
4369         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4370 }
4371
4372 static int create_var_field(struct hist_trigger_data *hist_data,
4373                             unsigned int val_idx,
4374                             struct trace_event_file *file,
4375                             char *var_name, char *expr_str)
4376 {
4377         unsigned long flags = 0;
4378
4379         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4380                 return -EINVAL;
4381
4382         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4383                 hist_err("Variable already defined: ", var_name);
4384                 return -EINVAL;
4385         }
4386
4387         flags |= HIST_FIELD_FL_VAR;
4388         hist_data->n_vars++;
4389         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4390                 return -EINVAL;
4391
4392         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4393 }
4394
4395 static int create_val_fields(struct hist_trigger_data *hist_data,
4396                              struct trace_event_file *file)
4397 {
4398         char *fields_str, *field_str;
4399         unsigned int i, j = 1;
4400         int ret;
4401
4402         ret = create_hitcount_val(hist_data);
4403         if (ret)
4404                 goto out;
4405
4406         fields_str = hist_data->attrs->vals_str;
4407         if (!fields_str)
4408                 goto out;
4409
4410         strsep(&fields_str, "=");
4411         if (!fields_str)
4412                 goto out;
4413
4414         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4415                      j < TRACING_MAP_VALS_MAX; i++) {
4416                 field_str = strsep(&fields_str, ",");
4417                 if (!field_str)
4418                         break;
4419
4420                 if (strcmp(field_str, "hitcount") == 0)
4421                         continue;
4422
4423                 ret = create_val_field(hist_data, j++, file, field_str);
4424                 if (ret)
4425                         goto out;
4426         }
4427
4428         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4429                 ret = -EINVAL;
4430  out:
4431         return ret;
4432 }
4433
4434 static int create_key_field(struct hist_trigger_data *hist_data,
4435                             unsigned int key_idx,
4436                             unsigned int key_offset,
4437                             struct trace_event_file *file,
4438                             char *field_str)
4439 {
4440         struct hist_field *hist_field = NULL;
4441
4442         unsigned long flags = 0;
4443         unsigned int key_size;
4444         int ret = 0;
4445
4446         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4447                 return -EINVAL;
4448
4449         flags |= HIST_FIELD_FL_KEY;
4450
4451         if (strcmp(field_str, "stacktrace") == 0) {
4452                 flags |= HIST_FIELD_FL_STACKTRACE;
4453                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4454                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4455         } else {
4456                 hist_field = parse_expr(hist_data, file, field_str, flags,
4457                                         NULL, 0);
4458                 if (IS_ERR(hist_field)) {
4459                         ret = PTR_ERR(hist_field);
4460                         goto out;
4461                 }
4462
4463                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
4464                         hist_err("Using variable references as keys not supported: ", field_str);
4465                         destroy_hist_field(hist_field, 0);
4466                         ret = -EINVAL;
4467                         goto out;
4468                 }
4469
4470                 key_size = hist_field->size;
4471         }
4472
4473         hist_data->fields[key_idx] = hist_field;
4474
4475         key_size = ALIGN(key_size, sizeof(u64));
4476         hist_data->fields[key_idx]->size = key_size;
4477         hist_data->fields[key_idx]->offset = key_offset;
4478
4479         hist_data->key_size += key_size;
4480
4481         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4482                 ret = -EINVAL;
4483                 goto out;
4484         }
4485
4486         hist_data->n_keys++;
4487         hist_data->n_fields++;
4488
4489         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4490                 return -EINVAL;
4491
4492         ret = key_size;
4493  out:
4494         return ret;
4495 }
4496
4497 static int create_key_fields(struct hist_trigger_data *hist_data,
4498                              struct trace_event_file *file)
4499 {
4500         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4501         char *fields_str, *field_str;
4502         int ret = -EINVAL;
4503
4504         fields_str = hist_data->attrs->keys_str;
4505         if (!fields_str)
4506                 goto out;
4507
4508         strsep(&fields_str, "=");
4509         if (!fields_str)
4510                 goto out;
4511
4512         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4513                 field_str = strsep(&fields_str, ",");
4514                 if (!field_str)
4515                         break;
4516                 ret = create_key_field(hist_data, i, key_offset,
4517                                        file, field_str);
4518                 if (ret < 0)
4519                         goto out;
4520                 key_offset += ret;
4521         }
4522         if (fields_str) {
4523                 ret = -EINVAL;
4524                 goto out;
4525         }
4526         ret = 0;
4527  out:
4528         return ret;
4529 }
4530
4531 static int create_var_fields(struct hist_trigger_data *hist_data,
4532                              struct trace_event_file *file)
4533 {
4534         unsigned int i, j = hist_data->n_vals;
4535         int ret = 0;
4536
4537         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4538
4539         for (i = 0; i < n_vars; i++) {
4540                 char *var_name = hist_data->attrs->var_defs.name[i];
4541                 char *expr = hist_data->attrs->var_defs.expr[i];
4542
4543                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4544                 if (ret)
4545                         goto out;
4546         }
4547  out:
4548         return ret;
4549 }
4550
4551 static void free_var_defs(struct hist_trigger_data *hist_data)
4552 {
4553         unsigned int i;
4554
4555         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4556                 kfree(hist_data->attrs->var_defs.name[i]);
4557                 kfree(hist_data->attrs->var_defs.expr[i]);
4558         }
4559
4560         hist_data->attrs->var_defs.n_vars = 0;
4561 }
4562
4563 static int parse_var_defs(struct hist_trigger_data *hist_data)
4564 {
4565         char *s, *str, *var_name, *field_str;
4566         unsigned int i, j, n_vars = 0;
4567         int ret = 0;
4568
4569         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4570                 str = hist_data->attrs->assignment_str[i];
4571                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4572                         field_str = strsep(&str, ",");
4573                         if (!field_str)
4574                                 break;
4575
4576                         var_name = strsep(&field_str, "=");
4577                         if (!var_name || !field_str) {
4578                                 hist_err("Malformed assignment: ", var_name);
4579                                 ret = -EINVAL;
4580                                 goto free;
4581                         }
4582
4583                         if (n_vars == TRACING_MAP_VARS_MAX) {
4584                                 hist_err("Too many variables defined: ", var_name);
4585                                 ret = -EINVAL;
4586                                 goto free;
4587                         }
4588
4589                         s = kstrdup(var_name, GFP_KERNEL);
4590                         if (!s) {
4591                                 ret = -ENOMEM;
4592                                 goto free;
4593                         }
4594                         hist_data->attrs->var_defs.name[n_vars] = s;
4595
4596                         s = kstrdup(field_str, GFP_KERNEL);
4597                         if (!s) {
4598                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4599                                 ret = -ENOMEM;
4600                                 goto free;
4601                         }
4602                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4603
4604                         hist_data->attrs->var_defs.n_vars = n_vars;
4605                 }
4606         }
4607
4608         return ret;
4609  free:
4610         free_var_defs(hist_data);
4611
4612         return ret;
4613 }
4614
4615 static int create_hist_fields(struct hist_trigger_data *hist_data,
4616                               struct trace_event_file *file)
4617 {
4618         int ret;
4619
4620         ret = parse_var_defs(hist_data);
4621         if (ret)
4622                 goto out;
4623
4624         ret = create_val_fields(hist_data, file);
4625         if (ret)
4626                 goto out;
4627
4628         ret = create_var_fields(hist_data, file);
4629         if (ret)
4630                 goto out;
4631
4632         ret = create_key_fields(hist_data, file);
4633         if (ret)
4634                 goto out;
4635  out:
4636         free_var_defs(hist_data);
4637
4638         return ret;
4639 }
4640
4641 static int is_descending(const char *str)
4642 {
4643         if (!str)
4644                 return 0;
4645
4646         if (strcmp(str, "descending") == 0)
4647                 return 1;
4648
4649         if (strcmp(str, "ascending") == 0)
4650                 return 0;
4651
4652         return -EINVAL;
4653 }
4654
4655 static int create_sort_keys(struct hist_trigger_data *hist_data)
4656 {
4657         char *fields_str = hist_data->attrs->sort_key_str;
4658         struct tracing_map_sort_key *sort_key;
4659         int descending, ret = 0;
4660         unsigned int i, j, k;
4661
4662         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4663
4664         if (!fields_str)
4665                 goto out;
4666
4667         strsep(&fields_str, "=");
4668         if (!fields_str) {
4669                 ret = -EINVAL;
4670                 goto out;
4671         }
4672
4673         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4674                 struct hist_field *hist_field;
4675                 char *field_str, *field_name;
4676                 const char *test_name;
4677
4678                 sort_key = &hist_data->sort_keys[i];
4679
4680                 field_str = strsep(&fields_str, ",");
4681                 if (!field_str) {
4682                         if (i == 0)
4683                                 ret = -EINVAL;
4684                         break;
4685                 }
4686
4687                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4688                         ret = -EINVAL;
4689                         break;
4690                 }
4691
4692                 field_name = strsep(&field_str, ".");
4693                 if (!field_name) {
4694                         ret = -EINVAL;
4695                         break;
4696                 }
4697
4698                 if (strcmp(field_name, "hitcount") == 0) {
4699                         descending = is_descending(field_str);
4700                         if (descending < 0) {
4701                                 ret = descending;
4702                                 break;
4703                         }
4704                         sort_key->descending = descending;
4705                         continue;
4706                 }
4707
4708                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4709                         unsigned int idx;
4710
4711                         hist_field = hist_data->fields[j];
4712                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4713                                 continue;
4714
4715                         idx = k++;
4716
4717                         test_name = hist_field_name(hist_field, 0);
4718
4719                         if (strcmp(field_name, test_name) == 0) {
4720                                 sort_key->field_idx = idx;
4721                                 descending = is_descending(field_str);
4722                                 if (descending < 0) {
4723                                         ret = descending;
4724                                         goto out;
4725                                 }
4726                                 sort_key->descending = descending;
4727                                 break;
4728                         }
4729                 }
4730                 if (j == hist_data->n_fields) {
4731                         ret = -EINVAL;
4732                         break;
4733                 }
4734         }
4735
4736         hist_data->n_sort_keys = i;
4737  out:
4738         return ret;
4739 }
4740
4741 static void destroy_actions(struct hist_trigger_data *hist_data)
4742 {
4743         unsigned int i;
4744
4745         for (i = 0; i < hist_data->n_actions; i++) {
4746                 struct action_data *data = hist_data->actions[i];
4747
4748                 if (data->handler == HANDLER_ONMATCH)
4749                         onmatch_destroy(data);
4750                 else if (data->handler == HANDLER_ONMAX ||
4751                          data->handler == HANDLER_ONCHANGE)
4752                         track_data_destroy(hist_data, data);
4753                 else
4754                         kfree(data);
4755         }
4756 }
4757
4758 static int parse_actions(struct hist_trigger_data *hist_data)
4759 {
4760         struct trace_array *tr = hist_data->event_file->tr;
4761         struct action_data *data;
4762         unsigned int i;
4763         int ret = 0;
4764         char *str;
4765         int len;
4766
4767         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4768                 str = hist_data->attrs->action_str[i];
4769
4770                 if ((len = str_has_prefix(str, "onmatch("))) {
4771                         char *action_str = str + len;
4772
4773                         data = onmatch_parse(tr, action_str);
4774                         if (IS_ERR(data)) {
4775                                 ret = PTR_ERR(data);
4776                                 break;
4777                         }
4778                 } else if ((len = str_has_prefix(str, "onmax("))) {
4779                         char *action_str = str + len;
4780
4781                         data = track_data_parse(hist_data, action_str,
4782                                                 HANDLER_ONMAX);
4783                         if (IS_ERR(data)) {
4784                                 ret = PTR_ERR(data);
4785                                 break;
4786                         }
4787                 } else if ((len = str_has_prefix(str, "onchange("))) {
4788                         char *action_str = str + len;
4789
4790                         data = track_data_parse(hist_data, action_str,
4791                                                 HANDLER_ONCHANGE);
4792                         if (IS_ERR(data)) {
4793                                 ret = PTR_ERR(data);
4794                                 break;
4795                         }
4796                 } else {
4797                         ret = -EINVAL;
4798                         break;
4799                 }
4800
4801                 hist_data->actions[hist_data->n_actions++] = data;
4802         }
4803
4804         return ret;
4805 }
4806
4807 static int create_actions(struct hist_trigger_data *hist_data)
4808 {
4809         struct action_data *data;
4810         unsigned int i;
4811         int ret = 0;
4812
4813         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4814                 data = hist_data->actions[i];
4815
4816                 if (data->handler == HANDLER_ONMATCH) {
4817                         ret = onmatch_create(hist_data, data);
4818                         if (ret)
4819                                 break;
4820                 } else if (data->handler == HANDLER_ONMAX ||
4821                            data->handler == HANDLER_ONCHANGE) {
4822                         ret = track_data_create(hist_data, data);
4823                         if (ret)
4824                                 break;
4825                 } else {
4826                         ret = -EINVAL;
4827                         break;
4828                 }
4829         }
4830
4831         return ret;
4832 }
4833
4834 static void print_actions(struct seq_file *m,
4835                           struct hist_trigger_data *hist_data,
4836                           struct tracing_map_elt *elt)
4837 {
4838         unsigned int i;
4839
4840         for (i = 0; i < hist_data->n_actions; i++) {
4841                 struct action_data *data = hist_data->actions[i];
4842
4843                 if (data->action == ACTION_SNAPSHOT)
4844                         continue;
4845
4846                 if (data->handler == HANDLER_ONMAX ||
4847                     data->handler == HANDLER_ONCHANGE)
4848                         track_data_print(m, hist_data, elt, data);
4849         }
4850 }
4851
4852 static void print_action_spec(struct seq_file *m,
4853                               struct hist_trigger_data *hist_data,
4854                               struct action_data *data)
4855 {
4856         unsigned int i;
4857
4858         if (data->action == ACTION_SAVE) {
4859                 for (i = 0; i < hist_data->n_save_vars; i++) {
4860                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4861                         if (i < hist_data->n_save_vars - 1)
4862                                 seq_puts(m, ",");
4863                 }
4864         } else if (data->action == ACTION_TRACE) {
4865                 if (data->use_trace_keyword)
4866                         seq_printf(m, "%s", data->synth_event_name);
4867                 for (i = 0; i < data->n_params; i++) {
4868                         if (i || data->use_trace_keyword)
4869                                 seq_puts(m, ",");
4870                         seq_printf(m, "%s", data->params[i]);
4871                 }
4872         }
4873 }
4874
4875 static void print_track_data_spec(struct seq_file *m,
4876                                   struct hist_trigger_data *hist_data,
4877                                   struct action_data *data)
4878 {
4879         if (data->handler == HANDLER_ONMAX)
4880                 seq_puts(m, ":onmax(");
4881         else if (data->handler == HANDLER_ONCHANGE)
4882                 seq_puts(m, ":onchange(");
4883         seq_printf(m, "%s", data->track_data.var_str);
4884         seq_printf(m, ").%s(", data->action_name);
4885
4886         print_action_spec(m, hist_data, data);
4887
4888         seq_puts(m, ")");
4889 }
4890
4891 static void print_onmatch_spec(struct seq_file *m,
4892                                struct hist_trigger_data *hist_data,
4893                                struct action_data *data)
4894 {
4895         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4896                    data->match_data.event);
4897
4898         seq_printf(m, "%s(", data->action_name);
4899
4900         print_action_spec(m, hist_data, data);
4901
4902         seq_puts(m, ")");
4903 }
4904
4905 static bool actions_match(struct hist_trigger_data *hist_data,
4906                           struct hist_trigger_data *hist_data_test)
4907 {
4908         unsigned int i, j;
4909
4910         if (hist_data->n_actions != hist_data_test->n_actions)
4911                 return false;
4912
4913         for (i = 0; i < hist_data->n_actions; i++) {
4914                 struct action_data *data = hist_data->actions[i];
4915                 struct action_data *data_test = hist_data_test->actions[i];
4916                 char *action_name, *action_name_test;
4917
4918                 if (data->handler != data_test->handler)
4919                         return false;
4920                 if (data->action != data_test->action)
4921                         return false;
4922
4923                 if (data->n_params != data_test->n_params)
4924                         return false;
4925
4926                 for (j = 0; j < data->n_params; j++) {
4927                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4928                                 return false;
4929                 }
4930
4931                 if (data->use_trace_keyword)
4932                         action_name = data->synth_event_name;
4933                 else
4934                         action_name = data->action_name;
4935
4936                 if (data_test->use_trace_keyword)
4937                         action_name_test = data_test->synth_event_name;
4938                 else
4939                         action_name_test = data_test->action_name;
4940
4941                 if (strcmp(action_name, action_name_test) != 0)
4942                         return false;
4943
4944                 if (data->handler == HANDLER_ONMATCH) {
4945                         if (strcmp(data->match_data.event_system,
4946                                    data_test->match_data.event_system) != 0)
4947                                 return false;
4948                         if (strcmp(data->match_data.event,
4949                                    data_test->match_data.event) != 0)
4950                                 return false;
4951                 } else if (data->handler == HANDLER_ONMAX ||
4952                            data->handler == HANDLER_ONCHANGE) {
4953                         if (strcmp(data->track_data.var_str,
4954                                    data_test->track_data.var_str) != 0)
4955                                 return false;
4956                 }
4957         }
4958
4959         return true;
4960 }
4961
4962
4963 static void print_actions_spec(struct seq_file *m,
4964                                struct hist_trigger_data *hist_data)
4965 {
4966         unsigned int i;
4967
4968         for (i = 0; i < hist_data->n_actions; i++) {
4969                 struct action_data *data = hist_data->actions[i];
4970
4971                 if (data->handler == HANDLER_ONMATCH)
4972                         print_onmatch_spec(m, hist_data, data);
4973                 else if (data->handler == HANDLER_ONMAX ||
4974                          data->handler == HANDLER_ONCHANGE)
4975                         print_track_data_spec(m, hist_data, data);
4976         }
4977 }
4978
4979 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4980 {
4981         unsigned int i;
4982
4983         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4984                 kfree(hist_data->field_var_hists[i]->cmd);
4985                 kfree(hist_data->field_var_hists[i]);
4986         }
4987 }
4988
4989 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4990 {
4991         if (!hist_data)
4992                 return;
4993
4994         destroy_hist_trigger_attrs(hist_data->attrs);
4995         destroy_hist_fields(hist_data);
4996         tracing_map_destroy(hist_data->map);
4997
4998         destroy_actions(hist_data);
4999         destroy_field_vars(hist_data);
5000         destroy_field_var_hists(hist_data);
5001
5002         kfree(hist_data);
5003 }
5004
5005 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5006 {
5007         struct tracing_map *map = hist_data->map;
5008         struct ftrace_event_field *field;
5009         struct hist_field *hist_field;
5010         int i, idx = 0;
5011
5012         for_each_hist_field(i, hist_data) {
5013                 hist_field = hist_data->fields[i];
5014                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5015                         tracing_map_cmp_fn_t cmp_fn;
5016
5017                         field = hist_field->field;
5018
5019                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5020                                 cmp_fn = tracing_map_cmp_none;
5021                         else if (!field)
5022                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5023                                                              hist_field->is_signed);
5024                         else if (is_string_field(field))
5025                                 cmp_fn = tracing_map_cmp_string;
5026                         else
5027                                 cmp_fn = tracing_map_cmp_num(field->size,
5028                                                              field->is_signed);
5029                         idx = tracing_map_add_key_field(map,
5030                                                         hist_field->offset,
5031                                                         cmp_fn);
5032                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5033                         idx = tracing_map_add_sum_field(map);
5034
5035                 if (idx < 0)
5036                         return idx;
5037
5038                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5039                         idx = tracing_map_add_var(map);
5040                         if (idx < 0)
5041                                 return idx;
5042                         hist_field->var.idx = idx;
5043                         hist_field->var.hist_data = hist_data;
5044                 }
5045         }
5046
5047         return 0;
5048 }
5049
5050 static struct hist_trigger_data *
5051 create_hist_data(unsigned int map_bits,
5052                  struct hist_trigger_attrs *attrs,
5053                  struct trace_event_file *file,
5054                  bool remove)
5055 {
5056         const struct tracing_map_ops *map_ops = NULL;
5057         struct hist_trigger_data *hist_data;
5058         int ret = 0;
5059
5060         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5061         if (!hist_data)
5062                 return ERR_PTR(-ENOMEM);
5063
5064         hist_data->attrs = attrs;
5065         hist_data->remove = remove;
5066         hist_data->event_file = file;
5067
5068         ret = parse_actions(hist_data);
5069         if (ret)
5070                 goto free;
5071
5072         ret = create_hist_fields(hist_data, file);
5073         if (ret)
5074                 goto free;
5075
5076         ret = create_sort_keys(hist_data);
5077         if (ret)
5078                 goto free;
5079
5080         map_ops = &hist_trigger_elt_data_ops;
5081
5082         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5083                                             map_ops, hist_data);
5084         if (IS_ERR(hist_data->map)) {
5085                 ret = PTR_ERR(hist_data->map);
5086                 hist_data->map = NULL;
5087                 goto free;
5088         }
5089
5090         ret = create_tracing_map_fields(hist_data);
5091         if (ret)
5092                 goto free;
5093  out:
5094         return hist_data;
5095  free:
5096         hist_data->attrs = NULL;
5097
5098         destroy_hist_data(hist_data);
5099
5100         hist_data = ERR_PTR(ret);
5101
5102         goto out;
5103 }
5104
5105 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5106                                     struct tracing_map_elt *elt, void *rec,
5107                                     struct ring_buffer_event *rbe,
5108                                     u64 *var_ref_vals)
5109 {
5110         struct hist_elt_data *elt_data;
5111         struct hist_field *hist_field;
5112         unsigned int i, var_idx;
5113         u64 hist_val;
5114
5115         elt_data = elt->private_data;
5116         elt_data->var_ref_vals = var_ref_vals;
5117
5118         for_each_hist_val_field(i, hist_data) {
5119                 hist_field = hist_data->fields[i];
5120                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5121                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5122                         var_idx = hist_field->var.idx;
5123                         tracing_map_set_var(elt, var_idx, hist_val);
5124                         continue;
5125                 }
5126                 tracing_map_update_sum(elt, i, hist_val);
5127         }
5128
5129         for_each_hist_key_field(i, hist_data) {
5130                 hist_field = hist_data->fields[i];
5131                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5132                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5133                         var_idx = hist_field->var.idx;
5134                         tracing_map_set_var(elt, var_idx, hist_val);
5135                 }
5136         }
5137
5138         update_field_vars(hist_data, elt, rbe, rec);
5139 }
5140
5141 static inline void add_to_key(char *compound_key, void *key,
5142                               struct hist_field *key_field, void *rec)
5143 {
5144         size_t size = key_field->size;
5145
5146         if (key_field->flags & HIST_FIELD_FL_STRING) {
5147                 struct ftrace_event_field *field;
5148
5149                 field = key_field->field;
5150                 if (field->filter_type == FILTER_DYN_STRING)
5151                         size = *(u32 *)(rec + field->offset) >> 16;
5152                 else if (field->filter_type == FILTER_PTR_STRING)
5153                         size = strlen(key);
5154                 else if (field->filter_type == FILTER_STATIC_STRING)
5155                         size = field->size;
5156
5157                 /* ensure NULL-termination */
5158                 if (size > key_field->size - 1)
5159                         size = key_field->size - 1;
5160
5161                 strncpy(compound_key + key_field->offset, (char *)key, size);
5162         } else
5163                 memcpy(compound_key + key_field->offset, key, size);
5164 }
5165
5166 static void
5167 hist_trigger_actions(struct hist_trigger_data *hist_data,
5168                      struct tracing_map_elt *elt, void *rec,
5169                      struct ring_buffer_event *rbe, void *key,
5170                      u64 *var_ref_vals)
5171 {
5172         struct action_data *data;
5173         unsigned int i;
5174
5175         for (i = 0; i < hist_data->n_actions; i++) {
5176                 data = hist_data->actions[i];
5177                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
5178         }
5179 }
5180
5181 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
5182                                struct ring_buffer_event *rbe)
5183 {
5184         struct hist_trigger_data *hist_data = data->private_data;
5185         bool use_compound_key = (hist_data->n_keys > 1);
5186         unsigned long entries[HIST_STACKTRACE_DEPTH];
5187         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5188         char compound_key[HIST_KEY_SIZE_MAX];
5189         struct tracing_map_elt *elt = NULL;
5190         struct stack_trace stacktrace;
5191         struct hist_field *key_field;
5192         u64 field_contents;
5193         void *key = NULL;
5194         unsigned int i;
5195
5196         memset(compound_key, 0, hist_data->key_size);
5197
5198         for_each_hist_key_field(i, hist_data) {
5199                 key_field = hist_data->fields[i];
5200
5201                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5202                         stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
5203                         stacktrace.entries = entries;
5204                         stacktrace.nr_entries = 0;
5205                         stacktrace.skip = HIST_STACKTRACE_SKIP;
5206
5207                         memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
5208                         save_stack_trace(&stacktrace);
5209
5210                         key = entries;
5211                 } else {
5212                         field_contents = key_field->fn(key_field, elt, rbe, rec);
5213                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5214                                 key = (void *)(unsigned long)field_contents;
5215                                 use_compound_key = true;
5216                         } else
5217                                 key = (void *)&field_contents;
5218                 }
5219
5220                 if (use_compound_key)
5221                         add_to_key(compound_key, key, key_field, rec);
5222         }
5223
5224         if (use_compound_key)
5225                 key = compound_key;
5226
5227         if (hist_data->n_var_refs &&
5228             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5229                 return;
5230
5231         elt = tracing_map_insert(hist_data->map, key);
5232         if (!elt)
5233                 return;
5234
5235         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
5236
5237         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5238                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
5239 }
5240
5241 static void hist_trigger_stacktrace_print(struct seq_file *m,
5242                                           unsigned long *stacktrace_entries,
5243                                           unsigned int max_entries)
5244 {
5245         char str[KSYM_SYMBOL_LEN];
5246         unsigned int spaces = 8;
5247         unsigned int i;
5248
5249         for (i = 0; i < max_entries; i++) {
5250                 if (stacktrace_entries[i] == ULONG_MAX)
5251                         return;
5252
5253                 seq_printf(m, "%*c", 1 + spaces, ' ');
5254                 sprint_symbol(str, stacktrace_entries[i]);
5255                 seq_printf(m, "%s\n", str);
5256         }
5257 }
5258
5259 static void hist_trigger_print_key(struct seq_file *m,
5260                                    struct hist_trigger_data *hist_data,
5261                                    void *key,
5262                                    struct tracing_map_elt *elt)
5263 {
5264         struct hist_field *key_field;
5265         char str[KSYM_SYMBOL_LEN];
5266         bool multiline = false;
5267         const char *field_name;
5268         unsigned int i;
5269         u64 uval;
5270
5271         seq_puts(m, "{ ");
5272
5273         for_each_hist_key_field(i, hist_data) {
5274                 key_field = hist_data->fields[i];
5275
5276                 if (i > hist_data->n_vals)
5277                         seq_puts(m, ", ");
5278
5279                 field_name = hist_field_name(key_field, 0);
5280
5281                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5282                         uval = *(u64 *)(key + key_field->offset);
5283                         seq_printf(m, "%s: %llx", field_name, uval);
5284                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5285                         uval = *(u64 *)(key + key_field->offset);
5286                         sprint_symbol_no_offset(str, uval);
5287                         seq_printf(m, "%s: [%llx] %-45s", field_name,
5288                                    uval, str);
5289                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5290                         uval = *(u64 *)(key + key_field->offset);
5291                         sprint_symbol(str, uval);
5292                         seq_printf(m, "%s: [%llx] %-55s", field_name,
5293                                    uval, str);
5294                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5295                         struct hist_elt_data *elt_data = elt->private_data;
5296                         char *comm;
5297
5298                         if (WARN_ON_ONCE(!elt_data))
5299                                 return;
5300
5301                         comm = elt_data->comm;
5302
5303                         uval = *(u64 *)(key + key_field->offset);
5304                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5305                                    comm, uval);
5306                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5307                         const char *syscall_name;
5308
5309                         uval = *(u64 *)(key + key_field->offset);
5310                         syscall_name = get_syscall_name(uval);
5311                         if (!syscall_name)
5312                                 syscall_name = "unknown_syscall";
5313
5314                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5315                                    syscall_name, uval);
5316                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5317                         seq_puts(m, "stacktrace:\n");
5318                         hist_trigger_stacktrace_print(m,
5319                                                       key + key_field->offset,
5320                                                       HIST_STACKTRACE_DEPTH);
5321                         multiline = true;
5322                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5323                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5324                                    *(u64 *)(key + key_field->offset));
5325                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5326                         seq_printf(m, "%s: %-50s", field_name,
5327                                    (char *)(key + key_field->offset));
5328                 } else {
5329                         uval = *(u64 *)(key + key_field->offset);
5330                         seq_printf(m, "%s: %10llu", field_name, uval);
5331                 }
5332         }
5333
5334         if (!multiline)
5335                 seq_puts(m, " ");
5336
5337         seq_puts(m, "}");
5338 }
5339
5340 static void hist_trigger_entry_print(struct seq_file *m,
5341                                      struct hist_trigger_data *hist_data,
5342                                      void *key,
5343                                      struct tracing_map_elt *elt)
5344 {
5345         const char *field_name;
5346         unsigned int i;
5347
5348         hist_trigger_print_key(m, hist_data, key, elt);
5349
5350         seq_printf(m, " hitcount: %10llu",
5351                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5352
5353         for (i = 1; i < hist_data->n_vals; i++) {
5354                 field_name = hist_field_name(hist_data->fields[i], 0);
5355
5356                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5357                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5358                         continue;
5359
5360                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5361                         seq_printf(m, "  %s: %10llx", field_name,
5362                                    tracing_map_read_sum(elt, i));
5363                 } else {
5364                         seq_printf(m, "  %s: %10llu", field_name,
5365                                    tracing_map_read_sum(elt, i));
5366                 }
5367         }
5368
5369         print_actions(m, hist_data, elt);
5370
5371         seq_puts(m, "\n");
5372 }
5373
5374 static int print_entries(struct seq_file *m,
5375                          struct hist_trigger_data *hist_data)
5376 {
5377         struct tracing_map_sort_entry **sort_entries = NULL;
5378         struct tracing_map *map = hist_data->map;
5379         int i, n_entries;
5380
5381         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5382                                              hist_data->n_sort_keys,
5383                                              &sort_entries);
5384         if (n_entries < 0)
5385                 return n_entries;
5386
5387         for (i = 0; i < n_entries; i++)
5388                 hist_trigger_entry_print(m, hist_data,
5389                                          sort_entries[i]->key,
5390                                          sort_entries[i]->elt);
5391
5392         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5393
5394         return n_entries;
5395 }
5396
5397 static void hist_trigger_show(struct seq_file *m,
5398                               struct event_trigger_data *data, int n)
5399 {
5400         struct hist_trigger_data *hist_data;
5401         int n_entries;
5402
5403         if (n > 0)
5404                 seq_puts(m, "\n\n");
5405
5406         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5407         data->ops->print(m, data->ops, data);
5408         seq_puts(m, "#\n\n");
5409
5410         hist_data = data->private_data;
5411         n_entries = print_entries(m, hist_data);
5412         if (n_entries < 0)
5413                 n_entries = 0;
5414
5415         track_data_snapshot_print(m, hist_data);
5416
5417         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5418                    (u64)atomic64_read(&hist_data->map->hits),
5419                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5420 }
5421
5422 static int hist_show(struct seq_file *m, void *v)
5423 {
5424         struct event_trigger_data *data;
5425         struct trace_event_file *event_file;
5426         int n = 0, ret = 0;
5427
5428         mutex_lock(&event_mutex);
5429
5430         event_file = event_file_data(m->private);
5431         if (unlikely(!event_file)) {
5432                 ret = -ENODEV;
5433                 goto out_unlock;
5434         }
5435
5436         list_for_each_entry_rcu(data, &event_file->triggers, list) {
5437                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5438                         hist_trigger_show(m, data, n++);
5439         }
5440
5441         if (have_hist_err()) {
5442                 seq_printf(m, "\nERROR: %s\n", hist_err_str);
5443                 seq_printf(m, "  Last command: %s\n", last_hist_cmd);
5444         }
5445
5446  out_unlock:
5447         mutex_unlock(&event_mutex);
5448
5449         return ret;
5450 }
5451
5452 static int event_hist_open(struct inode *inode, struct file *file)
5453 {
5454         return single_open(file, hist_show, file);
5455 }
5456
5457 const struct file_operations event_hist_fops = {
5458         .open = event_hist_open,
5459         .read = seq_read,
5460         .llseek = seq_lseek,
5461         .release = single_release,
5462 };
5463
5464 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5465 {
5466         const char *field_name = hist_field_name(hist_field, 0);
5467
5468         if (hist_field->var.name)
5469                 seq_printf(m, "%s=", hist_field->var.name);
5470
5471         if (hist_field->flags & HIST_FIELD_FL_CPU)
5472                 seq_puts(m, "cpu");
5473         else if (field_name) {
5474                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5475                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5476                         seq_putc(m, '$');
5477                 seq_printf(m, "%s", field_name);
5478         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5479                 seq_puts(m, "common_timestamp");
5480
5481         if (hist_field->flags) {
5482                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5483                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5484                         const char *flags = get_hist_field_flags(hist_field);
5485
5486                         if (flags)
5487                                 seq_printf(m, ".%s", flags);
5488                 }
5489         }
5490 }
5491
5492 static int event_hist_trigger_print(struct seq_file *m,
5493                                     struct event_trigger_ops *ops,
5494                                     struct event_trigger_data *data)
5495 {
5496         struct hist_trigger_data *hist_data = data->private_data;
5497         struct hist_field *field;
5498         bool have_var = false;
5499         unsigned int i;
5500
5501         seq_puts(m, "hist:");
5502
5503         if (data->name)
5504                 seq_printf(m, "%s:", data->name);
5505
5506         seq_puts(m, "keys=");
5507
5508         for_each_hist_key_field(i, hist_data) {
5509                 field = hist_data->fields[i];
5510
5511                 if (i > hist_data->n_vals)
5512                         seq_puts(m, ",");
5513
5514                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5515                         seq_puts(m, "stacktrace");
5516                 else
5517                         hist_field_print(m, field);
5518         }
5519
5520         seq_puts(m, ":vals=");
5521
5522         for_each_hist_val_field(i, hist_data) {
5523                 field = hist_data->fields[i];
5524                 if (field->flags & HIST_FIELD_FL_VAR) {
5525                         have_var = true;
5526                         continue;
5527                 }
5528
5529                 if (i == HITCOUNT_IDX)
5530                         seq_puts(m, "hitcount");
5531                 else {
5532                         seq_puts(m, ",");
5533                         hist_field_print(m, field);
5534                 }
5535         }
5536
5537         if (have_var) {
5538                 unsigned int n = 0;
5539
5540                 seq_puts(m, ":");
5541
5542                 for_each_hist_val_field(i, hist_data) {
5543                         field = hist_data->fields[i];
5544
5545                         if (field->flags & HIST_FIELD_FL_VAR) {
5546                                 if (n++)
5547                                         seq_puts(m, ",");
5548                                 hist_field_print(m, field);
5549                         }
5550                 }
5551         }
5552
5553         seq_puts(m, ":sort=");
5554
5555         for (i = 0; i < hist_data->n_sort_keys; i++) {
5556                 struct tracing_map_sort_key *sort_key;
5557                 unsigned int idx, first_key_idx;
5558
5559                 /* skip VAR vals */
5560                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5561
5562                 sort_key = &hist_data->sort_keys[i];
5563                 idx = sort_key->field_idx;
5564
5565                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5566                         return -EINVAL;
5567
5568                 if (i > 0)
5569                         seq_puts(m, ",");
5570
5571                 if (idx == HITCOUNT_IDX)
5572                         seq_puts(m, "hitcount");
5573                 else {
5574                         if (idx >= first_key_idx)
5575                                 idx += hist_data->n_vars;
5576                         hist_field_print(m, hist_data->fields[idx]);
5577                 }
5578
5579                 if (sort_key->descending)
5580                         seq_puts(m, ".descending");
5581         }
5582         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5583         if (hist_data->enable_timestamps)
5584                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5585
5586         print_actions_spec(m, hist_data);
5587
5588         if (data->filter_str)
5589                 seq_printf(m, " if %s", data->filter_str);
5590
5591         if (data->paused)
5592                 seq_puts(m, " [paused]");
5593         else
5594                 seq_puts(m, " [active]");
5595
5596         seq_putc(m, '\n');
5597
5598         return 0;
5599 }
5600
5601 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5602                                    struct event_trigger_data *data)
5603 {
5604         struct hist_trigger_data *hist_data = data->private_data;
5605
5606         if (!data->ref && hist_data->attrs->name)
5607                 save_named_trigger(hist_data->attrs->name, data);
5608
5609         data->ref++;
5610
5611         return 0;
5612 }
5613
5614 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5615 {
5616         struct trace_event_file *file;
5617         unsigned int i;
5618         char *cmd;
5619         int ret;
5620
5621         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5622                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5623                 cmd = hist_data->field_var_hists[i]->cmd;
5624                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5625                                               "!hist", "hist", cmd);
5626         }
5627 }
5628
5629 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5630                                     struct event_trigger_data *data)
5631 {
5632         struct hist_trigger_data *hist_data = data->private_data;
5633
5634         if (WARN_ON_ONCE(data->ref <= 0))
5635                 return;
5636
5637         data->ref--;
5638         if (!data->ref) {
5639                 if (data->name)
5640                         del_named_trigger(data);
5641
5642                 trigger_data_free(data);
5643
5644                 remove_hist_vars(hist_data);
5645
5646                 unregister_field_var_hists(hist_data);
5647
5648                 destroy_hist_data(hist_data);
5649         }
5650 }
5651
5652 static struct event_trigger_ops event_hist_trigger_ops = {
5653         .func                   = event_hist_trigger,
5654         .print                  = event_hist_trigger_print,
5655         .init                   = event_hist_trigger_init,
5656         .free                   = event_hist_trigger_free,
5657 };
5658
5659 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5660                                          struct event_trigger_data *data)
5661 {
5662         data->ref++;
5663
5664         save_named_trigger(data->named_data->name, data);
5665
5666         event_hist_trigger_init(ops, data->named_data);
5667
5668         return 0;
5669 }
5670
5671 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5672                                           struct event_trigger_data *data)
5673 {
5674         if (WARN_ON_ONCE(data->ref <= 0))
5675                 return;
5676
5677         event_hist_trigger_free(ops, data->named_data);
5678
5679         data->ref--;
5680         if (!data->ref) {
5681                 del_named_trigger(data);
5682                 trigger_data_free(data);
5683         }
5684 }
5685
5686 static struct event_trigger_ops event_hist_trigger_named_ops = {
5687         .func                   = event_hist_trigger,
5688         .print                  = event_hist_trigger_print,
5689         .init                   = event_hist_trigger_named_init,
5690         .free                   = event_hist_trigger_named_free,
5691 };
5692
5693 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5694                                                             char *param)
5695 {
5696         return &event_hist_trigger_ops;
5697 }
5698
5699 static void hist_clear(struct event_trigger_data *data)
5700 {
5701         struct hist_trigger_data *hist_data = data->private_data;
5702
5703         if (data->name)
5704                 pause_named_trigger(data);
5705
5706         tracepoint_synchronize_unregister();
5707
5708         tracing_map_clear(hist_data->map);
5709
5710         if (data->name)
5711                 unpause_named_trigger(data);
5712 }
5713
5714 static bool compatible_field(struct ftrace_event_field *field,
5715                              struct ftrace_event_field *test_field)
5716 {
5717         if (field == test_field)
5718                 return true;
5719         if (field == NULL || test_field == NULL)
5720                 return false;
5721         if (strcmp(field->name, test_field->name) != 0)
5722                 return false;
5723         if (strcmp(field->type, test_field->type) != 0)
5724                 return false;
5725         if (field->size != test_field->size)
5726                 return false;
5727         if (field->is_signed != test_field->is_signed)
5728                 return false;
5729
5730         return true;
5731 }
5732
5733 static bool hist_trigger_match(struct event_trigger_data *data,
5734                                struct event_trigger_data *data_test,
5735                                struct event_trigger_data *named_data,
5736                                bool ignore_filter)
5737 {
5738         struct tracing_map_sort_key *sort_key, *sort_key_test;
5739         struct hist_trigger_data *hist_data, *hist_data_test;
5740         struct hist_field *key_field, *key_field_test;
5741         unsigned int i;
5742
5743         if (named_data && (named_data != data_test) &&
5744             (named_data != data_test->named_data))
5745                 return false;
5746
5747         if (!named_data && is_named_trigger(data_test))
5748                 return false;
5749
5750         hist_data = data->private_data;
5751         hist_data_test = data_test->private_data;
5752
5753         if (hist_data->n_vals != hist_data_test->n_vals ||
5754             hist_data->n_fields != hist_data_test->n_fields ||
5755             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5756                 return false;
5757
5758         if (!ignore_filter) {
5759                 if ((data->filter_str && !data_test->filter_str) ||
5760                    (!data->filter_str && data_test->filter_str))
5761                         return false;
5762         }
5763
5764         for_each_hist_field(i, hist_data) {
5765                 key_field = hist_data->fields[i];
5766                 key_field_test = hist_data_test->fields[i];
5767
5768                 if (key_field->flags != key_field_test->flags)
5769                         return false;
5770                 if (!compatible_field(key_field->field, key_field_test->field))
5771                         return false;
5772                 if (key_field->offset != key_field_test->offset)
5773                         return false;
5774                 if (key_field->size != key_field_test->size)
5775                         return false;
5776                 if (key_field->is_signed != key_field_test->is_signed)
5777                         return false;
5778                 if (!!key_field->var.name != !!key_field_test->var.name)
5779                         return false;
5780                 if (key_field->var.name &&
5781                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5782                         return false;
5783         }
5784
5785         for (i = 0; i < hist_data->n_sort_keys; i++) {
5786                 sort_key = &hist_data->sort_keys[i];
5787                 sort_key_test = &hist_data_test->sort_keys[i];
5788
5789                 if (sort_key->field_idx != sort_key_test->field_idx ||
5790                     sort_key->descending != sort_key_test->descending)
5791                         return false;
5792         }
5793
5794         if (!ignore_filter && data->filter_str &&
5795             (strcmp(data->filter_str, data_test->filter_str) != 0))
5796                 return false;
5797
5798         if (!actions_match(hist_data, hist_data_test))
5799                 return false;
5800
5801         return true;
5802 }
5803
5804 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5805                                  struct event_trigger_data *data,
5806                                  struct trace_event_file *file)
5807 {
5808         struct hist_trigger_data *hist_data = data->private_data;
5809         struct event_trigger_data *test, *named_data = NULL;
5810         int ret = 0;
5811
5812         if (hist_data->attrs->name) {
5813                 named_data = find_named_trigger(hist_data->attrs->name);
5814                 if (named_data) {
5815                         if (!hist_trigger_match(data, named_data, named_data,
5816                                                 true)) {
5817                                 hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name);
5818                                 ret = -EINVAL;
5819                                 goto out;
5820                         }
5821                 }
5822         }
5823
5824         if (hist_data->attrs->name && !named_data)
5825                 goto new;
5826
5827         list_for_each_entry_rcu(test, &file->triggers, list) {
5828                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5829                         if (!hist_trigger_match(data, test, named_data, false))
5830                                 continue;
5831                         if (hist_data->attrs->pause)
5832                                 test->paused = true;
5833                         else if (hist_data->attrs->cont)
5834                                 test->paused = false;
5835                         else if (hist_data->attrs->clear)
5836                                 hist_clear(test);
5837                         else {
5838                                 hist_err("Hist trigger already exists", NULL);
5839                                 ret = -EEXIST;
5840                         }
5841                         goto out;
5842                 }
5843         }
5844  new:
5845         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5846                 hist_err("Can't clear or continue a nonexistent hist trigger", NULL);
5847                 ret = -ENOENT;
5848                 goto out;
5849         }
5850
5851         if (hist_data->attrs->pause)
5852                 data->paused = true;
5853
5854         if (named_data) {
5855                 data->private_data = named_data->private_data;
5856                 set_named_trigger_data(data, named_data);
5857                 data->ops = &event_hist_trigger_named_ops;
5858         }
5859
5860         if (data->ops->init) {
5861                 ret = data->ops->init(data->ops, data);
5862                 if (ret < 0)
5863                         goto out;
5864         }
5865
5866         if (hist_data->enable_timestamps) {
5867                 char *clock = hist_data->attrs->clock;
5868
5869                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5870                 if (ret) {
5871                         hist_err("Couldn't set trace_clock: ", clock);
5872                         goto out;
5873                 }
5874
5875                 tracing_set_time_stamp_abs(file->tr, true);
5876         }
5877
5878         if (named_data)
5879                 destroy_hist_data(hist_data);
5880
5881         ret++;
5882  out:
5883         return ret;
5884 }
5885
5886 static int hist_trigger_enable(struct event_trigger_data *data,
5887                                struct trace_event_file *file)
5888 {
5889         int ret = 0;
5890
5891         list_add_tail_rcu(&data->list, &file->triggers);
5892
5893         update_cond_flag(file);
5894
5895         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5896                 list_del_rcu(&data->list);
5897                 update_cond_flag(file);
5898                 ret--;
5899         }
5900
5901         return ret;
5902 }
5903
5904 static bool have_hist_trigger_match(struct event_trigger_data *data,
5905                                     struct trace_event_file *file)
5906 {
5907         struct hist_trigger_data *hist_data = data->private_data;
5908         struct event_trigger_data *test, *named_data = NULL;
5909         bool match = false;
5910
5911         if (hist_data->attrs->name)
5912                 named_data = find_named_trigger(hist_data->attrs->name);
5913
5914         list_for_each_entry_rcu(test, &file->triggers, list) {
5915                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5916                         if (hist_trigger_match(data, test, named_data, false)) {
5917                                 match = true;
5918                                 break;
5919                         }
5920                 }
5921         }
5922
5923         return match;
5924 }
5925
5926 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5927                                     struct trace_event_file *file)
5928 {
5929         struct hist_trigger_data *hist_data = data->private_data;
5930         struct event_trigger_data *test, *named_data = NULL;
5931
5932         if (hist_data->attrs->name)
5933                 named_data = find_named_trigger(hist_data->attrs->name);
5934
5935         list_for_each_entry_rcu(test, &file->triggers, list) {
5936                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5937                         if (!hist_trigger_match(data, test, named_data, false))
5938                                 continue;
5939                         hist_data = test->private_data;
5940                         if (check_var_refs(hist_data))
5941                                 return true;
5942                         break;
5943                 }
5944         }
5945
5946         return false;
5947 }
5948
5949 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5950                                     struct event_trigger_data *data,
5951                                     struct trace_event_file *file)
5952 {
5953         struct hist_trigger_data *hist_data = data->private_data;
5954         struct event_trigger_data *test, *named_data = NULL;
5955         bool unregistered = false;
5956
5957         if (hist_data->attrs->name)
5958                 named_data = find_named_trigger(hist_data->attrs->name);
5959
5960         list_for_each_entry_rcu(test, &file->triggers, list) {
5961                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5962                         if (!hist_trigger_match(data, test, named_data, false))
5963                                 continue;
5964                         unregistered = true;
5965                         list_del_rcu(&test->list);
5966                         trace_event_trigger_enable_disable(file, 0);
5967                         update_cond_flag(file);
5968                         break;
5969                 }
5970         }
5971
5972         if (unregistered && test->ops->free)
5973                 test->ops->free(test->ops, test);
5974
5975         if (hist_data->enable_timestamps) {
5976                 if (!hist_data->remove || unregistered)
5977                         tracing_set_time_stamp_abs(file->tr, false);
5978         }
5979 }
5980
5981 static bool hist_file_check_refs(struct trace_event_file *file)
5982 {
5983         struct hist_trigger_data *hist_data;
5984         struct event_trigger_data *test;
5985
5986         list_for_each_entry_rcu(test, &file->triggers, list) {
5987                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5988                         hist_data = test->private_data;
5989                         if (check_var_refs(hist_data))
5990                                 return true;
5991                 }
5992         }
5993
5994         return false;
5995 }
5996
5997 static void hist_unreg_all(struct trace_event_file *file)
5998 {
5999         struct event_trigger_data *test, *n;
6000         struct hist_trigger_data *hist_data;
6001         struct synth_event *se;
6002         const char *se_name;
6003
6004         lockdep_assert_held(&event_mutex);
6005
6006         if (hist_file_check_refs(file))
6007                 return;
6008
6009         list_for_each_entry_safe(test, n, &file->triggers, list) {
6010                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6011                         hist_data = test->private_data;
6012                         list_del_rcu(&test->list);
6013                         trace_event_trigger_enable_disable(file, 0);
6014
6015                         se_name = trace_event_name(file->event_call);
6016                         se = find_synth_event(se_name);
6017                         if (se)
6018                                 se->ref--;
6019
6020                         update_cond_flag(file);
6021                         if (hist_data->enable_timestamps)
6022                                 tracing_set_time_stamp_abs(file->tr, false);
6023                         if (test->ops->free)
6024                                 test->ops->free(test->ops, test);
6025                 }
6026         }
6027 }
6028
6029 static int event_hist_trigger_func(struct event_command *cmd_ops,
6030                                    struct trace_event_file *file,
6031                                    char *glob, char *cmd, char *param)
6032 {
6033         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6034         struct event_trigger_data *trigger_data;
6035         struct hist_trigger_attrs *attrs;
6036         struct event_trigger_ops *trigger_ops;
6037         struct hist_trigger_data *hist_data;
6038         struct synth_event *se;
6039         const char *se_name;
6040         bool remove = false;
6041         char *trigger, *p;
6042         int ret = 0;
6043
6044         lockdep_assert_held(&event_mutex);
6045
6046         if (glob && strlen(glob)) {
6047                 last_cmd_set(param);
6048                 hist_err_clear();
6049         }
6050
6051         if (!param)
6052                 return -EINVAL;
6053
6054         if (glob[0] == '!')
6055                 remove = true;
6056
6057         /*
6058          * separate the trigger from the filter (k:v [if filter])
6059          * allowing for whitespace in the trigger
6060          */
6061         p = trigger = param;
6062         do {
6063                 p = strstr(p, "if");
6064                 if (!p)
6065                         break;
6066                 if (p == param)
6067                         return -EINVAL;
6068                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6069                         p++;
6070                         continue;
6071                 }
6072                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6073                         return -EINVAL;
6074                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6075                         p++;
6076                         continue;
6077                 }
6078                 break;
6079         } while (p);
6080
6081         if (!p)
6082                 param = NULL;
6083         else {
6084                 *(p - 1) = '\0';
6085                 param = strstrip(p);
6086                 trigger = strstrip(trigger);
6087         }
6088
6089         attrs = parse_hist_trigger_attrs(trigger);
6090         if (IS_ERR(attrs))
6091                 return PTR_ERR(attrs);
6092
6093         if (attrs->map_bits)
6094                 hist_trigger_bits = attrs->map_bits;
6095
6096         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6097         if (IS_ERR(hist_data)) {
6098                 destroy_hist_trigger_attrs(attrs);
6099                 return PTR_ERR(hist_data);
6100         }
6101
6102         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6103
6104         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6105         if (!trigger_data) {
6106                 ret = -ENOMEM;
6107                 goto out_free;
6108         }
6109
6110         trigger_data->count = -1;
6111         trigger_data->ops = trigger_ops;
6112         trigger_data->cmd_ops = cmd_ops;
6113
6114         INIT_LIST_HEAD(&trigger_data->list);
6115         RCU_INIT_POINTER(trigger_data->filter, NULL);
6116
6117         trigger_data->private_data = hist_data;
6118
6119         /* if param is non-empty, it's supposed to be a filter */
6120         if (param && cmd_ops->set_filter) {
6121                 ret = cmd_ops->set_filter(param, trigger_data, file);
6122                 if (ret < 0)
6123                         goto out_free;
6124         }
6125
6126         if (remove) {
6127                 if (!have_hist_trigger_match(trigger_data, file))
6128                         goto out_free;
6129
6130                 if (hist_trigger_check_refs(trigger_data, file)) {
6131                         ret = -EBUSY;
6132                         goto out_free;
6133                 }
6134
6135                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6136                 se_name = trace_event_name(file->event_call);
6137                 se = find_synth_event(se_name);
6138                 if (se)
6139                         se->ref--;
6140                 ret = 0;
6141                 goto out_free;
6142         }
6143
6144         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6145         /*
6146          * The above returns on success the # of triggers registered,
6147          * but if it didn't register any it returns zero.  Consider no
6148          * triggers registered a failure too.
6149          */
6150         if (!ret) {
6151                 if (!(attrs->pause || attrs->cont || attrs->clear))
6152                         ret = -ENOENT;
6153                 goto out_free;
6154         } else if (ret < 0)
6155                 goto out_free;
6156
6157         if (get_named_trigger_data(trigger_data))
6158                 goto enable;
6159
6160         if (has_hist_vars(hist_data))
6161                 save_hist_vars(hist_data);
6162
6163         ret = create_actions(hist_data);
6164         if (ret)
6165                 goto out_unreg;
6166
6167         ret = tracing_map_init(hist_data->map);
6168         if (ret)
6169                 goto out_unreg;
6170 enable:
6171         ret = hist_trigger_enable(trigger_data, file);
6172         if (ret)
6173                 goto out_unreg;
6174
6175         se_name = trace_event_name(file->event_call);
6176         se = find_synth_event(se_name);
6177         if (se)
6178                 se->ref++;
6179         /* Just return zero, not the number of registered triggers */
6180         ret = 0;
6181  out:
6182         if (ret == 0)
6183                 hist_err_clear();
6184
6185         return ret;
6186  out_unreg:
6187         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6188  out_free:
6189         if (cmd_ops->set_filter)
6190                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6191
6192         remove_hist_vars(hist_data);
6193
6194         kfree(trigger_data);
6195
6196         destroy_hist_data(hist_data);
6197         goto out;
6198 }
6199
6200 static struct event_command trigger_hist_cmd = {
6201         .name                   = "hist",
6202         .trigger_type           = ETT_EVENT_HIST,
6203         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6204         .func                   = event_hist_trigger_func,
6205         .reg                    = hist_register_trigger,
6206         .unreg                  = hist_unregister_trigger,
6207         .unreg_all              = hist_unreg_all,
6208         .get_trigger_ops        = event_hist_get_trigger_ops,
6209         .set_filter             = set_trigger_filter,
6210 };
6211
6212 __init int register_trigger_hist_cmd(void)
6213 {
6214         int ret;
6215
6216         ret = register_event_command(&trigger_hist_cmd);
6217         WARN_ON(ret < 0);
6218
6219         return ret;
6220 }
6221
6222 static void
6223 hist_enable_trigger(struct event_trigger_data *data, void *rec,
6224                     struct ring_buffer_event *event)
6225 {
6226         struct enable_trigger_data *enable_data = data->private_data;
6227         struct event_trigger_data *test;
6228
6229         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
6230                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6231                         if (enable_data->enable)
6232                                 test->paused = false;
6233                         else
6234                                 test->paused = true;
6235                 }
6236         }
6237 }
6238
6239 static void
6240 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
6241                           struct ring_buffer_event *event)
6242 {
6243         if (!data->count)
6244                 return;
6245
6246         if (data->count != -1)
6247                 (data->count)--;
6248
6249         hist_enable_trigger(data, rec, event);
6250 }
6251
6252 static struct event_trigger_ops hist_enable_trigger_ops = {
6253         .func                   = hist_enable_trigger,
6254         .print                  = event_enable_trigger_print,
6255         .init                   = event_trigger_init,
6256         .free                   = event_enable_trigger_free,
6257 };
6258
6259 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6260         .func                   = hist_enable_count_trigger,
6261         .print                  = event_enable_trigger_print,
6262         .init                   = event_trigger_init,
6263         .free                   = event_enable_trigger_free,
6264 };
6265
6266 static struct event_trigger_ops hist_disable_trigger_ops = {
6267         .func                   = hist_enable_trigger,
6268         .print                  = event_enable_trigger_print,
6269         .init                   = event_trigger_init,
6270         .free                   = event_enable_trigger_free,
6271 };
6272
6273 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6274         .func                   = hist_enable_count_trigger,
6275         .print                  = event_enable_trigger_print,
6276         .init                   = event_trigger_init,
6277         .free                   = event_enable_trigger_free,
6278 };
6279
6280 static struct event_trigger_ops *
6281 hist_enable_get_trigger_ops(char *cmd, char *param)
6282 {
6283         struct event_trigger_ops *ops;
6284         bool enable;
6285
6286         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6287
6288         if (enable)
6289                 ops = param ? &hist_enable_count_trigger_ops :
6290                         &hist_enable_trigger_ops;
6291         else
6292                 ops = param ? &hist_disable_count_trigger_ops :
6293                         &hist_disable_trigger_ops;
6294
6295         return ops;
6296 }
6297
6298 static void hist_enable_unreg_all(struct trace_event_file *file)
6299 {
6300         struct event_trigger_data *test, *n;
6301
6302         list_for_each_entry_safe(test, n, &file->triggers, list) {
6303                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6304                         list_del_rcu(&test->list);
6305                         update_cond_flag(file);
6306                         trace_event_trigger_enable_disable(file, 0);
6307                         if (test->ops->free)
6308                                 test->ops->free(test->ops, test);
6309                 }
6310         }
6311 }
6312
6313 static struct event_command trigger_hist_enable_cmd = {
6314         .name                   = ENABLE_HIST_STR,
6315         .trigger_type           = ETT_HIST_ENABLE,
6316         .func                   = event_enable_trigger_func,
6317         .reg                    = event_enable_register_trigger,
6318         .unreg                  = event_enable_unregister_trigger,
6319         .unreg_all              = hist_enable_unreg_all,
6320         .get_trigger_ops        = hist_enable_get_trigger_ops,
6321         .set_filter             = set_trigger_filter,
6322 };
6323
6324 static struct event_command trigger_hist_disable_cmd = {
6325         .name                   = DISABLE_HIST_STR,
6326         .trigger_type           = ETT_HIST_ENABLE,
6327         .func                   = event_enable_trigger_func,
6328         .reg                    = event_enable_register_trigger,
6329         .unreg                  = event_enable_unregister_trigger,
6330         .unreg_all              = hist_enable_unreg_all,
6331         .get_trigger_ops        = hist_enable_get_trigger_ops,
6332         .set_filter             = set_trigger_filter,
6333 };
6334
6335 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6336 {
6337         unregister_event_command(&trigger_hist_enable_cmd);
6338         unregister_event_command(&trigger_hist_disable_cmd);
6339 }
6340
6341 __init int register_trigger_hist_enable_disable_cmds(void)
6342 {
6343         int ret;
6344
6345         ret = register_event_command(&trigger_hist_enable_cmd);
6346         if (WARN_ON(ret < 0))
6347                 return ret;
6348         ret = register_event_command(&trigger_hist_disable_cmd);
6349         if (WARN_ON(ret < 0))
6350                 unregister_trigger_hist_enable_disable_cmds();
6351
6352         return ret;
6353 }
6354
6355 static __init int trace_events_hist_init(void)
6356 {
6357         struct dentry *entry = NULL;
6358         struct dentry *d_tracer;
6359         int err = 0;
6360
6361         err = dyn_event_register(&synth_event_ops);
6362         if (err) {
6363                 pr_warn("Could not register synth_event_ops\n");
6364                 return err;
6365         }
6366
6367         d_tracer = tracing_init_dentry();
6368         if (IS_ERR(d_tracer)) {
6369                 err = PTR_ERR(d_tracer);
6370                 goto err;
6371         }
6372
6373         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
6374                                     NULL, &synth_events_fops);
6375         if (!entry) {
6376                 err = -ENODEV;
6377                 goto err;
6378         }
6379
6380         return err;
6381  err:
6382         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6383
6384         return err;
6385 }
6386
6387 fs_initcall(trace_events_hist_init);