kernel/trace/trace_events_hist.c

   1 /*
   2  * trace_events_hist - trace event hist triggers
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
  15  */
  16
  17 #include <linux/module.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/mutex.h>
  20 #include <linux/slab.h>
  21 #include <linux/stacktrace.h>
  22 #include <linux/rculist.h>
  23 #include <linux/tracefs.h>
  24
  25 #include "tracing_map.h"
  26 #include "trace.h"
  27
  28 #define SYNTH_SYSTEM            "synthetic"
  29 #define SYNTH_FIELDS_MAX        16
  30
  31 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  32
  33 struct hist_field;
  34
  35 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  36                                 struct tracing_map_elt *elt,
  37                                 struct ring_buffer_event *rbe,
  38                                 void *event);
  39
  40 #define HIST_FIELD_OPERANDS_MAX 2
  41 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  42 #define HIST_ACTIONS_MAX        8
  43
  44 enum field_op_id {
  45         FIELD_OP_NONE,
  46         FIELD_OP_PLUS,
  47         FIELD_OP_MINUS,
  48         FIELD_OP_UNARY_MINUS,
  49 };
  50
  51 struct hist_var {
  52         char                            *name;
  53         struct hist_trigger_data        *hist_data;
  54         unsigned int                    idx;
  55 };
  56
  57 struct hist_field {
  58         struct ftrace_event_field       *field;
  59         unsigned long                   flags;
  60         hist_field_fn_t                 fn;
  61         unsigned int                    size;
  62         unsigned int                    offset;
  63         unsigned int                    is_signed;
  64         const char                      *type;
  65         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
  66         struct hist_trigger_data        *hist_data;
  67         struct hist_var                 var;
  68         enum field_op_id                operator;
  69         char                            *system;
  70         char                            *event_name;
  71         char                            *name;
  72         unsigned int                    var_idx;
  73         unsigned int                    var_ref_idx;
  74         bool                            read_once;
  75 };
  76
  77 static u64 hist_field_none(struct hist_field *field,
  78                            struct tracing_map_elt *elt,
  79                            struct ring_buffer_event *rbe,
  80                            void *event)
  81 {
  82         return 0;
  83 }
  84
  85 static u64 hist_field_counter(struct hist_field *field,
  86                               struct tracing_map_elt *elt,
  87                               struct ring_buffer_event *rbe,
  88                               void *event)
  89 {
  90         return 1;
  91 }
  92
  93 static u64 hist_field_string(struct hist_field *hist_field,
  94                              struct tracing_map_elt *elt,
  95                              struct ring_buffer_event *rbe,
  96                              void *event)
  97 {
  98         char *addr = (char *)(event + hist_field->field->offset);
  99
 100         return (u64)(unsigned long)addr;
 101 }
 102
 103 static u64 hist_field_dynstring(struct hist_field *hist_field,
 104                                 struct tracing_map_elt *elt,
 105                                 struct ring_buffer_event *rbe,
 106                                 void *event)
 107 {
 108         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 109         int str_loc = str_item & 0xffff;
 110         char *addr = (char *)(event + str_loc);
 111
 112         return (u64)(unsigned long)addr;
 113 }
 114
 115 static u64 hist_field_pstring(struct hist_field *hist_field,
 116                               struct tracing_map_elt *elt,
 117                               struct ring_buffer_event *rbe,
 118                               void *event)
 119 {
 120         char **addr = (char **)(event + hist_field->field->offset);
 121
 122         return (u64)(unsigned long)*addr;
 123 }
 124
 125 static u64 hist_field_log2(struct hist_field *hist_field,
 126                            struct tracing_map_elt *elt,
 127                            struct ring_buffer_event *rbe,
 128                            void *event)
 129 {
 130         struct hist_field *operand = hist_field->operands[0];
 131
 132         u64 val = operand->fn(operand, elt, rbe, event);
 133
 134         return (u64) ilog2(roundup_pow_of_two(val));
 135 }
 136
 137 static u64 hist_field_plus(struct hist_field *hist_field,
 138                            struct tracing_map_elt *elt,
 139                            struct ring_buffer_event *rbe,
 140                            void *event)
 141 {
 142         struct hist_field *operand1 = hist_field->operands[0];
 143         struct hist_field *operand2 = hist_field->operands[1];
 144
 145         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 146         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 147
 148         return val1 + val2;
 149 }
 150
 151 static u64 hist_field_minus(struct hist_field *hist_field,
 152                             struct tracing_map_elt *elt,
 153                             struct ring_buffer_event *rbe,
 154                             void *event)
 155 {
 156         struct hist_field *operand1 = hist_field->operands[0];
 157         struct hist_field *operand2 = hist_field->operands[1];
 158
 159         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 160         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 161
 162         return val1 - val2;
 163 }
 164
 165 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 166                                   struct tracing_map_elt *elt,
 167                                   struct ring_buffer_event *rbe,
 168                                   void *event)
 169 {
 170         struct hist_field *operand = hist_field->operands[0];
 171
 172         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 173         u64 val = (u64)-sval;
 174
 175         return val;
 176 }
 177
 178 #define DEFINE_HIST_FIELD_FN(type)                                      \
 179         static u64 hist_field_##type(struct hist_field *hist_field,     \
 180                                      struct tracing_map_elt *elt,       \
 181                                      struct ring_buffer_event *rbe,     \
 182                                      void *event)                       \
 183 {                                                                       \
 184         type *addr = (type *)(event + hist_field->field->offset);       \
 185                                                                         \
 186         return (u64)(unsigned long)*addr;                               \
 187 }
 188
 189 DEFINE_HIST_FIELD_FN(s64);
 190 DEFINE_HIST_FIELD_FN(u64);
 191 DEFINE_HIST_FIELD_FN(s32);
 192 DEFINE_HIST_FIELD_FN(u32);
 193 DEFINE_HIST_FIELD_FN(s16);
 194 DEFINE_HIST_FIELD_FN(u16);
 195 DEFINE_HIST_FIELD_FN(s8);
 196 DEFINE_HIST_FIELD_FN(u8);
 197
 198 #define for_each_hist_field(i, hist_data)       \
 199         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 200
 201 #define for_each_hist_val_field(i, hist_data)   \
 202         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 203
 204 #define for_each_hist_key_field(i, hist_data)   \
 205         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 206
 207 #define HIST_STACKTRACE_DEPTH   16
 208 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 209 #define HIST_STACKTRACE_SKIP    5
 210
 211 #define HITCOUNT_IDX            0
 212 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 213
 214 enum hist_field_flags {
 215         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 216         HIST_FIELD_FL_KEY               = 1 << 1,
 217         HIST_FIELD_FL_STRING            = 1 << 2,
 218         HIST_FIELD_FL_HEX               = 1 << 3,
 219         HIST_FIELD_FL_SYM               = 1 << 4,
 220         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 221         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 222         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 223         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 224         HIST_FIELD_FL_LOG2              = 1 << 9,
 225         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 226         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 227         HIST_FIELD_FL_VAR               = 1 << 12,
 228         HIST_FIELD_FL_EXPR              = 1 << 13,
 229         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 230         HIST_FIELD_FL_CPU               = 1 << 15,
 231         HIST_FIELD_FL_ALIAS             = 1 << 16,
 232 };
 233
 234 struct var_defs {
 235         unsigned int    n_vars;
 236         char            *name[TRACING_MAP_VARS_MAX];
 237         char            *expr[TRACING_MAP_VARS_MAX];
 238 };
 239
 240 struct hist_trigger_attrs {
 241         char            *keys_str;
 242         char            *vals_str;
 243         char            *sort_key_str;
 244         char            *name;
 245         char            *clock;
 246         bool            pause;
 247         bool            cont;
 248         bool            clear;
 249         bool            ts_in_usecs;
 250         unsigned int    map_bits;
 251
 252         char            *assignment_str[TRACING_MAP_VARS_MAX];
 253         unsigned int    n_assignments;
 254
 255         char            *action_str[HIST_ACTIONS_MAX];
 256         unsigned int    n_actions;
 257
 258         struct var_defs var_defs;
 259 };
 260
 261 struct field_var {
 262         struct hist_field       *var;
 263         struct hist_field       *val;
 264 };
 265
 266 struct field_var_hist {
 267         struct hist_trigger_data        *hist_data;
 268         char                            *cmd;
 269 };
 270
 271 struct hist_trigger_data {
 272         struct hist_field               *fields[HIST_FIELDS_MAX];
 273         unsigned int                    n_vals;
 274         unsigned int                    n_keys;
 275         unsigned int                    n_fields;
 276         unsigned int                    n_vars;
 277         unsigned int                    key_size;
 278         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 279         unsigned int                    n_sort_keys;
 280         struct trace_event_file         *event_file;
 281         struct hist_trigger_attrs       *attrs;
 282         struct tracing_map              *map;
 283         bool                            enable_timestamps;
 284         bool                            remove;
 285         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 286         unsigned int                    n_var_refs;
 287
 288         struct action_data              *actions[HIST_ACTIONS_MAX];
 289         unsigned int                    n_actions;
 290
 291         struct hist_field               *synth_var_refs[SYNTH_FIELDS_MAX];
 292         unsigned int                    n_synth_var_refs;
 293         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 294         unsigned int                    n_field_vars;
 295         unsigned int                    n_field_var_str;
 296         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 297         unsigned int                    n_field_var_hists;
 298
 299         struct field_var                *max_vars[SYNTH_FIELDS_MAX];
 300         unsigned int                    n_max_vars;
 301         unsigned int                    n_max_var_str;
 302 };
 303
 304 struct synth_field {
 305         char *type;
 306         char *name;
 307         size_t size;
 308         bool is_signed;
 309         bool is_string;
 310 };
 311
 312 struct synth_event {
 313         struct list_head                        list;
 314         int                                     ref;
 315         char                                    *name;
 316         struct synth_field                      **fields;
 317         unsigned int                            n_fields;
 318         unsigned int                            n_u64;
 319         struct trace_event_class                class;
 320         struct trace_event_call                 call;
 321         struct tracepoint                       *tp;
 322 };
 323
 324 struct action_data;
 325
 326 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 327                              struct tracing_map_elt *elt, void *rec,
 328                              struct ring_buffer_event *rbe,
 329                              struct action_data *data, u64 *var_ref_vals);
 330
 331 struct action_data {
 332         action_fn_t             fn;
 333         unsigned int            n_params;
 334         char                    *params[SYNTH_FIELDS_MAX];
 335
 336         union {
 337                 struct {
 338                         unsigned int            var_ref_idx;
 339                         char                    *match_event;
 340                         char                    *match_event_system;
 341                         char                    *synth_event_name;
 342                         struct synth_event      *synth_event;
 343                 } onmatch;
 344
 345                 struct {
 346                         char                    *var_str;
 347                         char                    *fn_name;
 348                         unsigned int            max_var_ref_idx;
 349                         struct hist_field       *max_var;
 350                         struct hist_field       *var;
 351                 } onmax;
 352         };
 353 };
 354
 355
 356 static char last_hist_cmd[MAX_FILTER_STR_VAL];
 357 static char hist_err_str[MAX_FILTER_STR_VAL];
 358
 359 static void last_cmd_set(char *str)
 360 {
 361         if (!str)
 362                 return;
 363
 364         strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
 365 }
 366
 367 static void hist_err(char *str, char *var)
 368 {
 369         int maxlen = MAX_FILTER_STR_VAL - 1;
 370
 371         if (!str)
 372                 return;
 373
 374         if (strlen(hist_err_str))
 375                 return;
 376
 377         if (!var)
 378                 var = "";
 379
 380         if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
 381                 return;
 382
 383         strcat(hist_err_str, str);
 384         strcat(hist_err_str, var);
 385 }
 386
 387 static void hist_err_event(char *str, char *system, char *event, char *var)
 388 {
 389         char err[MAX_FILTER_STR_VAL];
 390
 391         if (system && var)
 392                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
 393         else if (system)
 394                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
 395         else
 396                 strncpy(err, var, MAX_FILTER_STR_VAL);
 397
 398         hist_err(str, err);
 399 }
 400
 401 static void hist_err_clear(void)
 402 {
 403         hist_err_str[0] = '\0';
 404 }
 405
 406 static bool have_hist_err(void)
 407 {
 408         if (strlen(hist_err_str))
 409                 return true;
 410
 411         return false;
 412 }
 413
 414 static LIST_HEAD(synth_event_list);
 415 static DEFINE_MUTEX(synth_event_mutex);
 416
 417 struct synth_trace_event {
 418         struct trace_entry      ent;
 419         u64                     fields[];
 420 };
 421
 422 static int synth_event_define_fields(struct trace_event_call *call)
 423 {
 424         struct synth_trace_event trace;
 425         int offset = offsetof(typeof(trace), fields);
 426         struct synth_event *event = call->data;
 427         unsigned int i, size, n_u64;
 428         char *name, *type;
 429         bool is_signed;
 430         int ret = 0;
 431
 432         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 433                 size = event->fields[i]->size;
 434                 is_signed = event->fields[i]->is_signed;
 435                 type = event->fields[i]->type;
 436                 name = event->fields[i]->name;
 437                 ret = trace_define_field(call, type, name, offset, size,
 438                                          is_signed, FILTER_OTHER);
 439                 if (ret)
 440                         break;
 441
 442                 if (event->fields[i]->is_string) {
 443                         offset += STR_VAR_LEN_MAX;
 444                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 445                 } else {
 446                         offset += sizeof(u64);
 447                         n_u64++;
 448                 }
 449         }
 450
 451         event->n_u64 = n_u64;
 452
 453         return ret;
 454 }
 455
 456 static bool synth_field_signed(char *type)
 457 {
 458         if (strncmp(type, "u", 1) == 0)
 459                 return false;
 460
 461         return true;
 462 }
 463
 464 static int synth_field_is_string(char *type)
 465 {
 466         if (strstr(type, "char[") != NULL)
 467                 return true;
 468
 469         return false;
 470 }
 471
 472 static int synth_field_string_size(char *type)
 473 {
 474         char buf[4], *end, *start;
 475         unsigned int len;
 476         int size, err;
 477
 478         start = strstr(type, "char[");
 479         if (start == NULL)
 480                 return -EINVAL;
 481         start += strlen("char[");
 482
 483         end = strchr(type, ']');
 484         if (!end || end < start)
 485                 return -EINVAL;
 486
 487         len = end - start;
 488         if (len > 3)
 489                 return -EINVAL;
 490
 491         strncpy(buf, start, len);
 492         buf[len] = '\0';
 493
 494         err = kstrtouint(buf, 0, &size);
 495         if (err)
 496                 return err;
 497
 498         if (size > STR_VAR_LEN_MAX)
 499                 return -EINVAL;
 500
 501         return size;
 502 }
 503
 504 static int synth_field_size(char *type)
 505 {
 506         int size = 0;
 507
 508         if (strcmp(type, "s64") == 0)
 509                 size = sizeof(s64);
 510         else if (strcmp(type, "u64") == 0)
 511                 size = sizeof(u64);
 512         else if (strcmp(type, "s32") == 0)
 513                 size = sizeof(s32);
 514         else if (strcmp(type, "u32") == 0)
 515                 size = sizeof(u32);
 516         else if (strcmp(type, "s16") == 0)
 517                 size = sizeof(s16);
 518         else if (strcmp(type, "u16") == 0)
 519                 size = sizeof(u16);
 520         else if (strcmp(type, "s8") == 0)
 521                 size = sizeof(s8);
 522         else if (strcmp(type, "u8") == 0)
 523                 size = sizeof(u8);
 524         else if (strcmp(type, "char") == 0)
 525                 size = sizeof(char);
 526         else if (strcmp(type, "unsigned char") == 0)
 527                 size = sizeof(unsigned char);
 528         else if (strcmp(type, "int") == 0)
 529                 size = sizeof(int);
 530         else if (strcmp(type, "unsigned int") == 0)
 531                 size = sizeof(unsigned int);
 532         else if (strcmp(type, "long") == 0)
 533                 size = sizeof(long);
 534         else if (strcmp(type, "unsigned long") == 0)
 535                 size = sizeof(unsigned long);
 536         else if (strcmp(type, "pid_t") == 0)
 537                 size = sizeof(pid_t);
 538         else if (synth_field_is_string(type))
 539                 size = synth_field_string_size(type);
 540
 541         return size;
 542 }
 543
 544 static const char *synth_field_fmt(char *type)
 545 {
 546         const char *fmt = "%llu";
 547
 548         if (strcmp(type, "s64") == 0)
 549                 fmt = "%lld";
 550         else if (strcmp(type, "u64") == 0)
 551                 fmt = "%llu";
 552         else if (strcmp(type, "s32") == 0)
 553                 fmt = "%d";
 554         else if (strcmp(type, "u32") == 0)
 555                 fmt = "%u";
 556         else if (strcmp(type, "s16") == 0)
 557                 fmt = "%d";
 558         else if (strcmp(type, "u16") == 0)
 559                 fmt = "%u";
 560         else if (strcmp(type, "s8") == 0)
 561                 fmt = "%d";
 562         else if (strcmp(type, "u8") == 0)
 563                 fmt = "%u";
 564         else if (strcmp(type, "char") == 0)
 565                 fmt = "%d";
 566         else if (strcmp(type, "unsigned char") == 0)
 567                 fmt = "%u";
 568         else if (strcmp(type, "int") == 0)
 569                 fmt = "%d";
 570         else if (strcmp(type, "unsigned int") == 0)
 571                 fmt = "%u";
 572         else if (strcmp(type, "long") == 0)
 573                 fmt = "%ld";
 574         else if (strcmp(type, "unsigned long") == 0)
 575                 fmt = "%lu";
 576         else if (strcmp(type, "pid_t") == 0)
 577                 fmt = "%d";
 578         else if (synth_field_is_string(type))
 579                 fmt = "%s";
 580
 581         return fmt;
 582 }
 583
 584 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 585                                            int flags,
 586                                            struct trace_event *event)
 587 {
 588         struct trace_array *tr = iter->tr;
 589         struct trace_seq *s = &iter->seq;
 590         struct synth_trace_event *entry;
 591         struct synth_event *se;
 592         unsigned int i, n_u64;
 593         char print_fmt[32];
 594         const char *fmt;
 595
 596         entry = (struct synth_trace_event *)iter->ent;
 597         se = container_of(event, struct synth_event, call.event);
 598
 599         trace_seq_printf(s, "%s: ", se->name);
 600
 601         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 602                 if (trace_seq_has_overflowed(s))
 603                         goto end;
 604
 605                 fmt = synth_field_fmt(se->fields[i]->type);
 606
 607                 /* parameter types */
 608                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 609                         trace_seq_printf(s, "%s ", fmt);
 610
 611                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 612
 613                 /* parameter values */
 614                 if (se->fields[i]->is_string) {
 615                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 616                                          (char *)&entry->fields[n_u64],
 617                                          i == se->n_fields - 1 ? "" : " ");
 618                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 619                 } else {
 620                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 621                                          entry->fields[n_u64],
 622                                          i == se->n_fields - 1 ? "" : " ");
 623                         n_u64++;
 624                 }
 625         }
 626 end:
 627         trace_seq_putc(s, '\n');
 628
 629         return trace_handle_return(s);
 630 }
 631
 632 static struct trace_event_functions synth_event_funcs = {
 633         .trace          = print_synth_event
 634 };
 635
 636 static notrace void trace_event_raw_event_synth(void *__data,
 637                                                 u64 *var_ref_vals,
 638                                                 unsigned int var_ref_idx)
 639 {
 640         struct trace_event_file *trace_file = __data;
 641         struct synth_trace_event *entry;
 642         struct trace_event_buffer fbuffer;
 643         struct ring_buffer *buffer;
 644         struct synth_event *event;
 645         unsigned int i, n_u64;
 646         int fields_size = 0;
 647
 648         event = trace_file->event_call->data;
 649
 650         if (trace_trigger_soft_disabled(trace_file))
 651                 return;
 652
 653         fields_size = event->n_u64 * sizeof(u64);
 654
 655         /*
 656          * Avoid ring buffer recursion detection, as this event
 657          * is being performed within another event.
 658          */
 659         buffer = trace_file->tr->trace_buffer.buffer;
 660         ring_buffer_nest_start(buffer);
 661
 662         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 663                                            sizeof(*entry) + fields_size);
 664         if (!entry)
 665                 goto out;
 666
 667         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 668                 if (event->fields[i]->is_string) {
 669                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 670                         char *str_field = (char *)&entry->fields[n_u64];
 671
 672                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 673                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 674                 } else {
 675                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 676                         n_u64++;
 677                 }
 678         }
 679
 680         trace_event_buffer_commit(&fbuffer);
 681 out:
 682         ring_buffer_nest_end(buffer);
 683 }
 684
 685 static void free_synth_event_print_fmt(struct trace_event_call *call)
 686 {
 687         if (call) {
 688                 kfree(call->print_fmt);
 689                 call->print_fmt = NULL;
 690         }
 691 }
 692
 693 static int __set_synth_event_print_fmt(struct synth_event *event,
 694                                        char *buf, int len)
 695 {
 696         const char *fmt;
 697         int pos = 0;
 698         int i;
 699
 700         /* When len=0, we just calculate the needed length */
 701 #define LEN_OR_ZERO (len ? len - pos : 0)
 702
 703         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 704         for (i = 0; i < event->n_fields; i++) {
 705                 fmt = synth_field_fmt(event->fields[i]->type);
 706                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 707                                 event->fields[i]->name, fmt,
 708                                 i == event->n_fields - 1 ? "" : ", ");
 709         }
 710         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 711
 712         for (i = 0; i < event->n_fields; i++) {
 713                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 714                                 ", REC->%s", event->fields[i]->name);
 715         }
 716
 717 #undef LEN_OR_ZERO
 718
 719         /* return the length of print_fmt */
 720         return pos;
 721 }
 722
 723 static int set_synth_event_print_fmt(struct trace_event_call *call)
 724 {
 725         struct synth_event *event = call->data;
 726         char *print_fmt;
 727         int len;
 728
 729         /* First: called with 0 length to calculate the needed length */
 730         len = __set_synth_event_print_fmt(event, NULL, 0);
 731
 732         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 733         if (!print_fmt)
 734                 return -ENOMEM;
 735
 736         /* Second: actually write the @print_fmt */
 737         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 738         call->print_fmt = print_fmt;
 739
 740         return 0;
 741 }
 742
 743 static void free_synth_field(struct synth_field *field)
 744 {
 745         kfree(field->type);
 746         kfree(field->name);
 747         kfree(field);
 748 }
 749
 750 static struct synth_field *parse_synth_field(char *field_type,
 751                                              char *field_name)
 752 {
 753         struct synth_field *field;
 754         int len, ret = 0;
 755         char *array;
 756
 757         if (field_type[0] == ';')
 758                 field_type++;
 759
 760         len = strlen(field_name);
 761         if (field_name[len - 1] == ';')
 762                 field_name[len - 1] = '\0';
 763
 764         field = kzalloc(sizeof(*field), GFP_KERNEL);
 765         if (!field)
 766                 return ERR_PTR(-ENOMEM);
 767
 768         len = strlen(field_type) + 1;
 769         array = strchr(field_name, '[');
 770         if (array)
 771                 len += strlen(array);
 772         field->type = kzalloc(len, GFP_KERNEL);
 773         if (!field->type) {
 774                 ret = -ENOMEM;
 775                 goto free;
 776         }
 777         strcat(field->type, field_type);
 778         if (array) {
 779                 strcat(field->type, array);
 780                 *array = '\0';
 781         }
 782
 783         field->size = synth_field_size(field->type);
 784         if (!field->size) {
 785                 ret = -EINVAL;
 786                 goto free;
 787         }
 788
 789         if (synth_field_is_string(field->type))
 790                 field->is_string = true;
 791
 792         field->is_signed = synth_field_signed(field->type);
 793
 794         field->name = kstrdup(field_name, GFP_KERNEL);
 795         if (!field->name) {
 796                 ret = -ENOMEM;
 797                 goto free;
 798         }
 799  out:
 800         return field;
 801  free:
 802         free_synth_field(field);
 803         field = ERR_PTR(ret);
 804         goto out;
 805 }
 806
 807 static void free_synth_tracepoint(struct tracepoint *tp)
 808 {
 809         if (!tp)
 810                 return;
 811
 812         kfree(tp->name);
 813         kfree(tp);
 814 }
 815
 816 static struct tracepoint *alloc_synth_tracepoint(char *name)
 817 {
 818         struct tracepoint *tp;
 819
 820         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 821         if (!tp)
 822                 return ERR_PTR(-ENOMEM);
 823
 824         tp->name = kstrdup(name, GFP_KERNEL);
 825         if (!tp->name) {
 826                 kfree(tp);
 827                 return ERR_PTR(-ENOMEM);
 828         }
 829
 830         return tp;
 831 }
 832
 833 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 834                                     unsigned int var_ref_idx);
 835
 836 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 837                                unsigned int var_ref_idx)
 838 {
 839         struct tracepoint *tp = event->tp;
 840
 841         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 842                 struct tracepoint_func *probe_func_ptr;
 843                 synth_probe_func_t probe_func;
 844                 void *__data;
 845
 846                 if (!(cpu_online(raw_smp_processor_id())))
 847                         return;
 848
 849                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 850                 if (probe_func_ptr) {
 851                         do {
 852                                 probe_func = probe_func_ptr->func;
 853                                 __data = probe_func_ptr->data;
 854                                 probe_func(__data, var_ref_vals, var_ref_idx);
 855                         } while ((++probe_func_ptr)->func);
 856                 }
 857         }
 858 }
 859
 860 static struct synth_event *find_synth_event(const char *name)
 861 {
 862         struct synth_event *event;
 863
 864         list_for_each_entry(event, &synth_event_list, list) {
 865                 if (strcmp(event->name, name) == 0)
 866                         return event;
 867         }
 868
 869         return NULL;
 870 }
 871
 872 static int register_synth_event(struct synth_event *event)
 873 {
 874         struct trace_event_call *call = &event->call;
 875         int ret = 0;
 876
 877         event->call.class = &event->class;
 878         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
 879         if (!event->class.system) {
 880                 ret = -ENOMEM;
 881                 goto out;
 882         }
 883
 884         event->tp = alloc_synth_tracepoint(event->name);
 885         if (IS_ERR(event->tp)) {
 886                 ret = PTR_ERR(event->tp);
 887                 event->tp = NULL;
 888                 goto out;
 889         }
 890
 891         INIT_LIST_HEAD(&call->class->fields);
 892         call->event.funcs = &synth_event_funcs;
 893         call->class->define_fields = synth_event_define_fields;
 894
 895         ret = register_trace_event(&call->event);
 896         if (!ret) {
 897                 ret = -ENODEV;
 898                 goto out;
 899         }
 900         call->flags = TRACE_EVENT_FL_TRACEPOINT;
 901         call->class->reg = trace_event_reg;
 902         call->class->probe = trace_event_raw_event_synth;
 903         call->data = event;
 904         call->tp = event->tp;
 905
 906         ret = trace_add_event_call(call);
 907         if (ret) {
 908                 pr_warn("Failed to register synthetic event: %s\n",
 909                         trace_event_name(call));
 910                 goto err;
 911         }
 912
 913         ret = set_synth_event_print_fmt(call);
 914         if (ret < 0) {
 915                 trace_remove_event_call(call);
 916                 goto err;
 917         }
 918  out:
 919         return ret;
 920  err:
 921         unregister_trace_event(&call->event);
 922         goto out;
 923 }
 924
 925 static int unregister_synth_event(struct synth_event *event)
 926 {
 927         struct trace_event_call *call = &event->call;
 928         int ret;
 929
 930         ret = trace_remove_event_call(call);
 931
 932         return ret;
 933 }
 934
 935 static void free_synth_event(struct synth_event *event)
 936 {
 937         unsigned int i;
 938
 939         if (!event)
 940                 return;
 941
 942         for (i = 0; i < event->n_fields; i++)
 943                 free_synth_field(event->fields[i]);
 944
 945         kfree(event->fields);
 946         kfree(event->name);
 947         kfree(event->class.system);
 948         free_synth_tracepoint(event->tp);
 949         free_synth_event_print_fmt(&event->call);
 950         kfree(event);
 951 }
 952
 953 static struct synth_event *alloc_synth_event(char *event_name, int n_fields,
 954                                              struct synth_field **fields)
 955 {
 956         struct synth_event *event;
 957         unsigned int i;
 958
 959         event = kzalloc(sizeof(*event), GFP_KERNEL);
 960         if (!event) {
 961                 event = ERR_PTR(-ENOMEM);
 962                 goto out;
 963         }
 964
 965         event->name = kstrdup(event_name, GFP_KERNEL);
 966         if (!event->name) {
 967                 kfree(event);
 968                 event = ERR_PTR(-ENOMEM);
 969                 goto out;
 970         }
 971
 972         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
 973         if (!event->fields) {
 974                 free_synth_event(event);
 975                 event = ERR_PTR(-ENOMEM);
 976                 goto out;
 977         }
 978
 979         for (i = 0; i < n_fields; i++)
 980                 event->fields[i] = fields[i];
 981
 982         event->n_fields = n_fields;
 983  out:
 984         return event;
 985 }
 986
 987 static void action_trace(struct hist_trigger_data *hist_data,
 988                          struct tracing_map_elt *elt, void *rec,
 989                          struct ring_buffer_event *rbe,
 990                          struct action_data *data, u64 *var_ref_vals)
 991 {
 992         struct synth_event *event = data->onmatch.synth_event;
 993
 994         trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx);
 995 }
 996
 997 struct hist_var_data {
 998         struct list_head list;
 999         struct hist_trigger_data *hist_data;
1000 };
1001
1002 static void add_or_delete_synth_event(struct synth_event *event, int delete)
1003 {
1004         if (delete)
1005                 free_synth_event(event);
1006         else {
1007                 mutex_lock(&synth_event_mutex);
1008                 if (!find_synth_event(event->name))
1009                         list_add(&event->list, &synth_event_list);
1010                 else
1011                         free_synth_event(event);
1012                 mutex_unlock(&synth_event_mutex);
1013         }
1014 }
1015
1016 static int create_synth_event(int argc, char **argv)
1017 {
1018         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1019         struct synth_event *event = NULL;
1020         bool delete_event = false;
1021         int i, n_fields = 0, ret = 0;
1022         char *name;
1023
1024         mutex_lock(&synth_event_mutex);
1025
1026         /*
1027          * Argument syntax:
1028          *  - Add synthetic event: <event_name> field[;field] ...
1029          *  - Remove synthetic event: !<event_name> field[;field] ...
1030          *      where 'field' = type field_name
1031          */
1032         if (argc < 1) {
1033                 ret = -EINVAL;
1034                 goto out;
1035         }
1036
1037         name = argv[0];
1038         if (name[0] == '!') {
1039                 delete_event = true;
1040                 name++;
1041         }
1042
1043         event = find_synth_event(name);
1044         if (event) {
1045                 if (delete_event) {
1046                         if (event->ref) {
1047                                 event = NULL;
1048                                 ret = -EBUSY;
1049                                 goto out;
1050                         }
1051                         list_del(&event->list);
1052                         goto out;
1053                 }
1054                 event = NULL;
1055                 ret = -EEXIST;
1056                 goto out;
1057         } else if (delete_event)
1058                 goto out;
1059
1060         if (argc < 2) {
1061                 ret = -EINVAL;
1062                 goto out;
1063         }
1064
1065         for (i = 1; i < argc - 1; i++) {
1066                 if (strcmp(argv[i], ";") == 0)
1067                         continue;
1068                 if (n_fields == SYNTH_FIELDS_MAX) {
1069                         ret = -EINVAL;
1070                         goto err;
1071                 }
1072
1073                 field = parse_synth_field(argv[i], argv[i + 1]);
1074                 if (IS_ERR(field)) {
1075                         ret = PTR_ERR(field);
1076                         goto err;
1077                 }
1078                 fields[n_fields] = field;
1079                 i++; n_fields++;
1080         }
1081
1082         if (i < argc) {
1083                 ret = -EINVAL;
1084                 goto err;
1085         }
1086
1087         event = alloc_synth_event(name, n_fields, fields);
1088         if (IS_ERR(event)) {
1089                 ret = PTR_ERR(event);
1090                 event = NULL;
1091                 goto err;
1092         }
1093  out:
1094         mutex_unlock(&synth_event_mutex);
1095
1096         if (event) {
1097                 if (delete_event) {
1098                         ret = unregister_synth_event(event);
1099                         add_or_delete_synth_event(event, !ret);
1100                 } else {
1101                         ret = register_synth_event(event);
1102                         add_or_delete_synth_event(event, ret);
1103                 }
1104         }
1105
1106         return ret;
1107  err:
1108         mutex_unlock(&synth_event_mutex);
1109
1110         for (i = 0; i < n_fields; i++)
1111                 free_synth_field(fields[i]);
1112         free_synth_event(event);
1113
1114         return ret;
1115 }
1116
1117 static int release_all_synth_events(void)
1118 {
1119         struct list_head release_events;
1120         struct synth_event *event, *e;
1121         int ret = 0;
1122
1123         INIT_LIST_HEAD(&release_events);
1124
1125         mutex_lock(&synth_event_mutex);
1126
1127         list_for_each_entry(event, &synth_event_list, list) {
1128                 if (event->ref) {
1129                         mutex_unlock(&synth_event_mutex);
1130                         return -EBUSY;
1131                 }
1132         }
1133
1134         list_splice_init(&event->list, &release_events);
1135
1136         mutex_unlock(&synth_event_mutex);
1137
1138         list_for_each_entry_safe(event, e, &release_events, list) {
1139                 list_del(&event->list);
1140
1141                 ret = unregister_synth_event(event);
1142                 add_or_delete_synth_event(event, !ret);
1143         }
1144
1145         return ret;
1146 }
1147
1148
1149 static void *synth_events_seq_start(struct seq_file *m, loff_t *pos)
1150 {
1151         mutex_lock(&synth_event_mutex);
1152
1153         return seq_list_start(&synth_event_list, *pos);
1154 }
1155
1156 static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos)
1157 {
1158         return seq_list_next(v, &synth_event_list, pos);
1159 }
1160
1161 static void synth_events_seq_stop(struct seq_file *m, void *v)
1162 {
1163         mutex_unlock(&synth_event_mutex);
1164 }
1165
1166 static int synth_events_seq_show(struct seq_file *m, void *v)
1167 {
1168         struct synth_field *field;
1169         struct synth_event *event = v;
1170         unsigned int i;
1171
1172         seq_printf(m, "%s\t", event->name);
1173
1174         for (i = 0; i < event->n_fields; i++) {
1175                 field = event->fields[i];
1176
1177                 /* parameter values */
1178                 seq_printf(m, "%s %s%s", field->type, field->name,
1179                            i == event->n_fields - 1 ? "" : "; ");
1180         }
1181
1182         seq_putc(m, '\n');
1183
1184         return 0;
1185 }
1186
1187 static const struct seq_operations synth_events_seq_op = {
1188         .start  = synth_events_seq_start,
1189         .next   = synth_events_seq_next,
1190         .stop   = synth_events_seq_stop,
1191         .show   = synth_events_seq_show
1192 };
1193
1194 static int synth_events_open(struct inode *inode, struct file *file)
1195 {
1196         int ret;
1197
1198         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1199                 ret = release_all_synth_events();
1200                 if (ret < 0)
1201                         return ret;
1202         }
1203
1204         return seq_open(file, &synth_events_seq_op);
1205 }
1206
1207 static ssize_t synth_events_write(struct file *file,
1208                                   const char __user *buffer,
1209                                   size_t count, loff_t *ppos)
1210 {
1211         return trace_parse_run_command(file, buffer, count, ppos,
1212                                        create_synth_event);
1213 }
1214
1215 static const struct file_operations synth_events_fops = {
1216         .open           = synth_events_open,
1217         .write          = synth_events_write,
1218         .read           = seq_read,
1219         .llseek         = seq_lseek,
1220         .release        = seq_release,
1221 };
1222
1223 static u64 hist_field_timestamp(struct hist_field *hist_field,
1224                                 struct tracing_map_elt *elt,
1225                                 struct ring_buffer_event *rbe,
1226                                 void *event)
1227 {
1228         struct hist_trigger_data *hist_data = hist_field->hist_data;
1229         struct trace_array *tr = hist_data->event_file->tr;
1230
1231         u64 ts = ring_buffer_event_time_stamp(rbe);
1232
1233         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1234                 ts = ns2usecs(ts);
1235
1236         return ts;
1237 }
1238
1239 static u64 hist_field_cpu(struct hist_field *hist_field,
1240                           struct tracing_map_elt *elt,
1241                           struct ring_buffer_event *rbe,
1242                           void *event)
1243 {
1244         int cpu = smp_processor_id();
1245
1246         return cpu;
1247 }
1248
1249 static struct hist_field *
1250 check_field_for_var_ref(struct hist_field *hist_field,
1251                         struct hist_trigger_data *var_data,
1252                         unsigned int var_idx)
1253 {
1254         struct hist_field *found = NULL;
1255
1256         if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) {
1257                 if (hist_field->var.idx == var_idx &&
1258                     hist_field->var.hist_data == var_data) {
1259                         found = hist_field;
1260                 }
1261         }
1262
1263         return found;
1264 }
1265
1266 static struct hist_field *
1267 check_field_for_var_refs(struct hist_trigger_data *hist_data,
1268                          struct hist_field *hist_field,
1269                          struct hist_trigger_data *var_data,
1270                          unsigned int var_idx,
1271                          unsigned int level)
1272 {
1273         struct hist_field *found = NULL;
1274         unsigned int i;
1275
1276         if (level > 3)
1277                 return found;
1278
1279         if (!hist_field)
1280                 return found;
1281
1282         found = check_field_for_var_ref(hist_field, var_data, var_idx);
1283         if (found)
1284                 return found;
1285
1286         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1287                 struct hist_field *operand;
1288
1289                 operand = hist_field->operands[i];
1290                 found = check_field_for_var_refs(hist_data, operand, var_data,
1291                                                  var_idx, level + 1);
1292                 if (found)
1293                         return found;
1294         }
1295
1296         return found;
1297 }
1298
1299 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1300                                        struct hist_trigger_data *var_data,
1301                                        unsigned int var_idx)
1302 {
1303         struct hist_field *hist_field, *found = NULL;
1304         unsigned int i;
1305
1306         for_each_hist_field(i, hist_data) {
1307                 hist_field = hist_data->fields[i];
1308                 found = check_field_for_var_refs(hist_data, hist_field,
1309                                                  var_data, var_idx, 0);
1310                 if (found)
1311                         return found;
1312         }
1313
1314         for (i = 0; i < hist_data->n_synth_var_refs; i++) {
1315                 hist_field = hist_data->synth_var_refs[i];
1316                 found = check_field_for_var_refs(hist_data, hist_field,
1317                                                  var_data, var_idx, 0);
1318                 if (found)
1319                         return found;
1320         }
1321
1322         return found;
1323 }
1324
1325 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1326                                            unsigned int var_idx)
1327 {
1328         struct trace_array *tr = hist_data->event_file->tr;
1329         struct hist_field *found = NULL;
1330         struct hist_var_data *var_data;
1331
1332         list_for_each_entry(var_data, &tr->hist_vars, list) {
1333                 if (var_data->hist_data == hist_data)
1334                         continue;
1335                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1336                 if (found)
1337                         break;
1338         }
1339
1340         return found;
1341 }
1342
1343 static bool check_var_refs(struct hist_trigger_data *hist_data)
1344 {
1345         struct hist_field *field;
1346         bool found = false;
1347         int i;
1348
1349         for_each_hist_field(i, hist_data) {
1350                 field = hist_data->fields[i];
1351                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1352                         if (find_any_var_ref(hist_data, field->var.idx)) {
1353                                 found = true;
1354                                 break;
1355                         }
1356                 }
1357         }
1358
1359         return found;
1360 }
1361
1362 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1363 {
1364         struct trace_array *tr = hist_data->event_file->tr;
1365         struct hist_var_data *var_data, *found = NULL;
1366
1367         list_for_each_entry(var_data, &tr->hist_vars, list) {
1368                 if (var_data->hist_data == hist_data) {
1369                         found = var_data;
1370                         break;
1371                 }
1372         }
1373
1374         return found;
1375 }
1376
1377 static bool field_has_hist_vars(struct hist_field *hist_field,
1378                                 unsigned int level)
1379 {
1380         int i;
1381
1382         if (level > 3)
1383                 return false;
1384
1385         if (!hist_field)
1386                 return false;
1387
1388         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1389             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1390                 return true;
1391
1392         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1393                 struct hist_field *operand;
1394
1395                 operand = hist_field->operands[i];
1396                 if (field_has_hist_vars(operand, level + 1))
1397                         return true;
1398         }
1399
1400         return false;
1401 }
1402
1403 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1404 {
1405         struct hist_field *hist_field;
1406         int i;
1407
1408         for_each_hist_field(i, hist_data) {
1409                 hist_field = hist_data->fields[i];
1410                 if (field_has_hist_vars(hist_field, 0))
1411                         return true;
1412         }
1413
1414         return false;
1415 }
1416
1417 static int save_hist_vars(struct hist_trigger_data *hist_data)
1418 {
1419         struct trace_array *tr = hist_data->event_file->tr;
1420         struct hist_var_data *var_data;
1421
1422         var_data = find_hist_vars(hist_data);
1423         if (var_data)
1424                 return 0;
1425
1426         if (trace_array_get(tr) < 0)
1427                 return -ENODEV;
1428
1429         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1430         if (!var_data) {
1431                 trace_array_put(tr);
1432                 return -ENOMEM;
1433         }
1434
1435         var_data->hist_data = hist_data;
1436         list_add(&var_data->list, &tr->hist_vars);
1437
1438         return 0;
1439 }
1440
1441 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1442 {
1443         struct trace_array *tr = hist_data->event_file->tr;
1444         struct hist_var_data *var_data;
1445
1446         var_data = find_hist_vars(hist_data);
1447         if (!var_data)
1448                 return;
1449
1450         if (WARN_ON(check_var_refs(hist_data)))
1451                 return;
1452
1453         list_del(&var_data->list);
1454
1455         kfree(var_data);
1456
1457         trace_array_put(tr);
1458 }
1459
1460 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1461                                          const char *var_name)
1462 {
1463         struct hist_field *hist_field, *found = NULL;
1464         int i;
1465
1466         for_each_hist_field(i, hist_data) {
1467                 hist_field = hist_data->fields[i];
1468                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1469                     strcmp(hist_field->var.name, var_name) == 0) {
1470                         found = hist_field;
1471                         break;
1472                 }
1473         }
1474
1475         return found;
1476 }
1477
1478 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1479                                    struct trace_event_file *file,
1480                                    const char *var_name)
1481 {
1482         struct hist_trigger_data *test_data;
1483         struct event_trigger_data *test;
1484         struct hist_field *hist_field;
1485
1486         hist_field = find_var_field(hist_data, var_name);
1487         if (hist_field)
1488                 return hist_field;
1489
1490         list_for_each_entry_rcu(test, &file->triggers, list) {
1491                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1492                         test_data = test->private_data;
1493                         hist_field = find_var_field(test_data, var_name);
1494                         if (hist_field)
1495                                 return hist_field;
1496                 }
1497         }
1498
1499         return NULL;
1500 }
1501
1502 static struct trace_event_file *find_var_file(struct trace_array *tr,
1503                                               char *system,
1504                                               char *event_name,
1505                                               char *var_name)
1506 {
1507         struct hist_trigger_data *var_hist_data;
1508         struct hist_var_data *var_data;
1509         struct trace_event_file *file, *found = NULL;
1510
1511         if (system)
1512                 return find_event_file(tr, system, event_name);
1513
1514         list_for_each_entry(var_data, &tr->hist_vars, list) {
1515                 var_hist_data = var_data->hist_data;
1516                 file = var_hist_data->event_file;
1517                 if (file == found)
1518                         continue;
1519
1520                 if (find_var_field(var_hist_data, var_name)) {
1521                         if (found) {
1522                                 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1523                                 return NULL;
1524                         }
1525
1526                         found = file;
1527                 }
1528         }
1529
1530         return found;
1531 }
1532
1533 static struct hist_field *find_file_var(struct trace_event_file *file,
1534                                         const char *var_name)
1535 {
1536         struct hist_trigger_data *test_data;
1537         struct event_trigger_data *test;
1538         struct hist_field *hist_field;
1539
1540         list_for_each_entry_rcu(test, &file->triggers, list) {
1541                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1542                         test_data = test->private_data;
1543                         hist_field = find_var_field(test_data, var_name);
1544                         if (hist_field)
1545                                 return hist_field;
1546                 }
1547         }
1548
1549         return NULL;
1550 }
1551
1552 static struct hist_field *
1553 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1554 {
1555         struct trace_array *tr = hist_data->event_file->tr;
1556         struct hist_field *hist_field, *found = NULL;
1557         struct trace_event_file *file;
1558         unsigned int i;
1559
1560         for (i = 0; i < hist_data->n_actions; i++) {
1561                 struct action_data *data = hist_data->actions[i];
1562
1563                 if (data->fn == action_trace) {
1564                         char *system = data->onmatch.match_event_system;
1565                         char *event_name = data->onmatch.match_event;
1566
1567                         file = find_var_file(tr, system, event_name, var_name);
1568                         if (!file)
1569                                 continue;
1570                         hist_field = find_file_var(file, var_name);
1571                         if (hist_field) {
1572                                 if (found) {
1573                                         hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1574                                         return ERR_PTR(-EINVAL);
1575                                 }
1576
1577                                 found = hist_field;
1578                         }
1579                 }
1580         }
1581         return found;
1582 }
1583
1584 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1585                                          char *system,
1586                                          char *event_name,
1587                                          char *var_name)
1588 {
1589         struct trace_array *tr = hist_data->event_file->tr;
1590         struct hist_field *hist_field = NULL;
1591         struct trace_event_file *file;
1592
1593         if (!system || !event_name) {
1594                 hist_field = find_match_var(hist_data, var_name);
1595                 if (IS_ERR(hist_field))
1596                         return NULL;
1597                 if (hist_field)
1598                         return hist_field;
1599         }
1600
1601         file = find_var_file(tr, system, event_name, var_name);
1602         if (!file)
1603                 return NULL;
1604
1605         hist_field = find_file_var(file, var_name);
1606
1607         return hist_field;
1608 }
1609
1610 struct hist_elt_data {
1611         char *comm;
1612         u64 *var_ref_vals;
1613         char *field_var_str[SYNTH_FIELDS_MAX];
1614 };
1615
1616 static u64 hist_field_var_ref(struct hist_field *hist_field,
1617                               struct tracing_map_elt *elt,
1618                               struct ring_buffer_event *rbe,
1619                               void *event)
1620 {
1621         struct hist_elt_data *elt_data;
1622         u64 var_val = 0;
1623
1624         elt_data = elt->private_data;
1625         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1626
1627         return var_val;
1628 }
1629
1630 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1631                              u64 *var_ref_vals, bool self)
1632 {
1633         struct hist_trigger_data *var_data;
1634         struct tracing_map_elt *var_elt;
1635         struct hist_field *hist_field;
1636         unsigned int i, var_idx;
1637         bool resolved = true;
1638         u64 var_val = 0;
1639
1640         for (i = 0; i < hist_data->n_var_refs; i++) {
1641                 hist_field = hist_data->var_refs[i];
1642                 var_idx = hist_field->var.idx;
1643                 var_data = hist_field->var.hist_data;
1644
1645                 if (var_data == NULL) {
1646                         resolved = false;
1647                         break;
1648                 }
1649
1650                 if ((self && var_data != hist_data) ||
1651                     (!self && var_data == hist_data))
1652                         continue;
1653
1654                 var_elt = tracing_map_lookup(var_data->map, key);
1655                 if (!var_elt) {
1656                         resolved = false;
1657                         break;
1658                 }
1659
1660                 if (!tracing_map_var_set(var_elt, var_idx)) {
1661                         resolved = false;
1662                         break;
1663                 }
1664
1665                 if (self || !hist_field->read_once)
1666                         var_val = tracing_map_read_var(var_elt, var_idx);
1667                 else
1668                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1669
1670                 var_ref_vals[i] = var_val;
1671         }
1672
1673         return resolved;
1674 }
1675
1676 static const char *hist_field_name(struct hist_field *field,
1677                                    unsigned int level)
1678 {
1679         const char *field_name = "";
1680
1681         if (level > 1)
1682                 return field_name;
1683
1684         if (field->field)
1685                 field_name = field->field->name;
1686         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1687                  field->flags & HIST_FIELD_FL_ALIAS)
1688                 field_name = hist_field_name(field->operands[0], ++level);
1689         else if (field->flags & HIST_FIELD_FL_CPU)
1690                 field_name = "cpu";
1691         else if (field->flags & HIST_FIELD_FL_EXPR ||
1692                  field->flags & HIST_FIELD_FL_VAR_REF) {
1693                 if (field->system) {
1694                         static char full_name[MAX_FILTER_STR_VAL];
1695
1696                         strcat(full_name, field->system);
1697                         strcat(full_name, ".");
1698                         strcat(full_name, field->event_name);
1699                         strcat(full_name, ".");
1700                         strcat(full_name, field->name);
1701                         field_name = full_name;
1702                 } else
1703                         field_name = field->name;
1704         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1705                 field_name = "common_timestamp";
1706
1707         if (field_name == NULL)
1708                 field_name = "";
1709
1710         return field_name;
1711 }
1712
1713 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1714 {
1715         hist_field_fn_t fn = NULL;
1716
1717         switch (field_size) {
1718         case 8:
1719                 if (field_is_signed)
1720                         fn = hist_field_s64;
1721                 else
1722                         fn = hist_field_u64;
1723                 break;
1724         case 4:
1725                 if (field_is_signed)
1726                         fn = hist_field_s32;
1727                 else
1728                         fn = hist_field_u32;
1729                 break;
1730         case 2:
1731                 if (field_is_signed)
1732                         fn = hist_field_s16;
1733                 else
1734                         fn = hist_field_u16;
1735                 break;
1736         case 1:
1737                 if (field_is_signed)
1738                         fn = hist_field_s8;
1739                 else
1740                         fn = hist_field_u8;
1741                 break;
1742         }
1743
1744         return fn;
1745 }
1746
1747 static int parse_map_size(char *str)
1748 {
1749         unsigned long size, map_bits;
1750         int ret;
1751
1752         strsep(&str, "=");
1753         if (!str) {
1754                 ret = -EINVAL;
1755                 goto out;
1756         }
1757
1758         ret = kstrtoul(str, 0, &size);
1759         if (ret)
1760                 goto out;
1761
1762         map_bits = ilog2(roundup_pow_of_two(size));
1763         if (map_bits < TRACING_MAP_BITS_MIN ||
1764             map_bits > TRACING_MAP_BITS_MAX)
1765                 ret = -EINVAL;
1766         else
1767                 ret = map_bits;
1768  out:
1769         return ret;
1770 }
1771
1772 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1773 {
1774         unsigned int i;
1775
1776         if (!attrs)
1777                 return;
1778
1779         for (i = 0; i < attrs->n_assignments; i++)
1780                 kfree(attrs->assignment_str[i]);
1781
1782         for (i = 0; i < attrs->n_actions; i++)
1783                 kfree(attrs->action_str[i]);
1784
1785         kfree(attrs->name);
1786         kfree(attrs->sort_key_str);
1787         kfree(attrs->keys_str);
1788         kfree(attrs->vals_str);
1789         kfree(attrs->clock);
1790         kfree(attrs);
1791 }
1792
1793 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1794 {
1795         int ret = -EINVAL;
1796
1797         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1798                 return ret;
1799
1800         if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) ||
1801             (strncmp(str, "onmax(", strlen("onmax(")) == 0)) {
1802                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1803                 if (!attrs->action_str[attrs->n_actions]) {
1804                         ret = -ENOMEM;
1805                         return ret;
1806                 }
1807                 attrs->n_actions++;
1808                 ret = 0;
1809         }
1810
1811         return ret;
1812 }
1813
1814 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1815 {
1816         int ret = 0;
1817
1818         if ((strncmp(str, "key=", strlen("key=")) == 0) ||
1819             (strncmp(str, "keys=", strlen("keys=")) == 0)) {
1820                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
1821                 if (!attrs->keys_str) {
1822                         ret = -ENOMEM;
1823                         goto out;
1824                 }
1825         } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
1826                  (strncmp(str, "vals=", strlen("vals=")) == 0) ||
1827                  (strncmp(str, "values=", strlen("values=")) == 0)) {
1828                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
1829                 if (!attrs->vals_str) {
1830                         ret = -ENOMEM;
1831                         goto out;
1832                 }
1833         } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
1834                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1835                 if (!attrs->sort_key_str) {
1836                         ret = -ENOMEM;
1837                         goto out;
1838                 }
1839         } else if (strncmp(str, "name=", strlen("name=")) == 0) {
1840                 attrs->name = kstrdup(str, GFP_KERNEL);
1841                 if (!attrs->name) {
1842                         ret = -ENOMEM;
1843                         goto out;
1844                 }
1845         } else if (strncmp(str, "clock=", strlen("clock=")) == 0) {
1846                 strsep(&str, "=");
1847                 if (!str) {
1848                         ret = -EINVAL;
1849                         goto out;
1850                 }
1851
1852                 str = strstrip(str);
1853                 attrs->clock = kstrdup(str, GFP_KERNEL);
1854                 if (!attrs->clock) {
1855                         ret = -ENOMEM;
1856                         goto out;
1857                 }
1858         } else if (strncmp(str, "size=", strlen("size=")) == 0) {
1859                 int map_bits = parse_map_size(str);
1860
1861                 if (map_bits < 0) {
1862                         ret = map_bits;
1863                         goto out;
1864                 }
1865                 attrs->map_bits = map_bits;
1866         } else {
1867                 char *assignment;
1868
1869                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1870                         hist_err("Too many variables defined: ", str);
1871                         ret = -EINVAL;
1872                         goto out;
1873                 }
1874
1875                 assignment = kstrdup(str, GFP_KERNEL);
1876                 if (!assignment) {
1877                         ret = -ENOMEM;
1878                         goto out;
1879                 }
1880
1881                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1882         }
1883  out:
1884         return ret;
1885 }
1886
1887 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
1888 {
1889         struct hist_trigger_attrs *attrs;
1890         int ret = 0;
1891
1892         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1893         if (!attrs)
1894                 return ERR_PTR(-ENOMEM);
1895
1896         while (trigger_str) {
1897                 char *str = strsep(&trigger_str, ":");
1898
1899                 if (strchr(str, '=')) {
1900                         ret = parse_assignment(str, attrs);
1901                         if (ret)
1902                                 goto free;
1903                 } else if (strcmp(str, "pause") == 0)
1904                         attrs->pause = true;
1905                 else if ((strcmp(str, "cont") == 0) ||
1906                          (strcmp(str, "continue") == 0))
1907                         attrs->cont = true;
1908                 else if (strcmp(str, "clear") == 0)
1909                         attrs->clear = true;
1910                 else {
1911                         ret = parse_action(str, attrs);
1912                         if (ret)
1913                                 goto free;
1914                 }
1915         }
1916
1917         if (!attrs->keys_str) {
1918                 ret = -EINVAL;
1919                 goto free;
1920         }
1921
1922         if (!attrs->clock) {
1923                 attrs->clock = kstrdup("global", GFP_KERNEL);
1924                 if (!attrs->clock) {
1925                         ret = -ENOMEM;
1926                         goto free;
1927                 }
1928         }
1929
1930         return attrs;
1931  free:
1932         destroy_hist_trigger_attrs(attrs);
1933
1934         return ERR_PTR(ret);
1935 }
1936
1937 static inline void save_comm(char *comm, struct task_struct *task)
1938 {
1939         if (!task->pid) {
1940                 strcpy(comm, "<idle>");
1941                 return;
1942         }
1943
1944         if (WARN_ON_ONCE(task->pid < 0)) {
1945                 strcpy(comm, "<XXX>");
1946                 return;
1947         }
1948
1949         memcpy(comm, task->comm, TASK_COMM_LEN);
1950 }
1951
1952 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1953 {
1954         unsigned int i;
1955
1956         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1957                 kfree(elt_data->field_var_str[i]);
1958
1959         kfree(elt_data->comm);
1960         kfree(elt_data);
1961 }
1962
1963 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1964 {
1965         struct hist_elt_data *elt_data = elt->private_data;
1966
1967         hist_elt_data_free(elt_data);
1968 }
1969
1970 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1971 {
1972         struct hist_trigger_data *hist_data = elt->map->private_data;
1973         unsigned int size = TASK_COMM_LEN;
1974         struct hist_elt_data *elt_data;
1975         struct hist_field *key_field;
1976         unsigned int i, n_str;
1977
1978         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1979         if (!elt_data)
1980                 return -ENOMEM;
1981
1982         for_each_hist_key_field(i, hist_data) {
1983                 key_field = hist_data->fields[i];
1984
1985                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1986                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1987                         if (!elt_data->comm) {
1988                                 kfree(elt_data);
1989                                 return -ENOMEM;
1990                         }
1991                         break;
1992                 }
1993         }
1994
1995         n_str = hist_data->n_field_var_str + hist_data->n_max_var_str;
1996
1997         size = STR_VAR_LEN_MAX;
1998
1999         for (i = 0; i < n_str; i++) {
2000                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2001                 if (!elt_data->field_var_str[i]) {
2002                         hist_elt_data_free(elt_data);
2003                         return -ENOMEM;
2004                 }
2005         }
2006
2007         elt->private_data = elt_data;
2008
2009         return 0;
2010 }
2011
2012 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2013 {
2014         struct hist_elt_data *elt_data = elt->private_data;
2015
2016         if (elt_data->comm)
2017                 save_comm(elt_data->comm, current);
2018 }
2019
2020 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2021         .elt_alloc      = hist_trigger_elt_data_alloc,
2022         .elt_free       = hist_trigger_elt_data_free,
2023         .elt_init       = hist_trigger_elt_data_init,
2024 };
2025
2026 static const char *get_hist_field_flags(struct hist_field *hist_field)
2027 {
2028         const char *flags_str = NULL;
2029
2030         if (hist_field->flags & HIST_FIELD_FL_HEX)
2031                 flags_str = "hex";
2032         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2033                 flags_str = "sym";
2034         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2035                 flags_str = "sym-offset";
2036         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2037                 flags_str = "execname";
2038         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2039                 flags_str = "syscall";
2040         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2041                 flags_str = "log2";
2042         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2043                 flags_str = "usecs";
2044
2045         return flags_str;
2046 }
2047
2048 static void expr_field_str(struct hist_field *field, char *expr)
2049 {
2050         if (field->flags & HIST_FIELD_FL_VAR_REF)
2051                 strcat(expr, "$");
2052
2053         strcat(expr, hist_field_name(field, 0));
2054
2055         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2056                 const char *flags_str = get_hist_field_flags(field);
2057
2058                 if (flags_str) {
2059                         strcat(expr, ".");
2060                         strcat(expr, flags_str);
2061                 }
2062         }
2063 }
2064
2065 static char *expr_str(struct hist_field *field, unsigned int level)
2066 {
2067         char *expr;
2068
2069         if (level > 1)
2070                 return NULL;
2071
2072         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2073         if (!expr)
2074                 return NULL;
2075
2076         if (!field->operands[0]) {
2077                 expr_field_str(field, expr);
2078                 return expr;
2079         }
2080
2081         if (field->operator == FIELD_OP_UNARY_MINUS) {
2082                 char *subexpr;
2083
2084                 strcat(expr, "-(");
2085                 subexpr = expr_str(field->operands[0], ++level);
2086                 if (!subexpr) {
2087                         kfree(expr);
2088                         return NULL;
2089                 }
2090                 strcat(expr, subexpr);
2091                 strcat(expr, ")");
2092
2093                 kfree(subexpr);
2094
2095                 return expr;
2096         }
2097
2098         expr_field_str(field->operands[0], expr);
2099
2100         switch (field->operator) {
2101         case FIELD_OP_MINUS:
2102                 strcat(expr, "-");
2103                 break;
2104         case FIELD_OP_PLUS:
2105                 strcat(expr, "+");
2106                 break;
2107         default:
2108                 kfree(expr);
2109                 return NULL;
2110         }
2111
2112         expr_field_str(field->operands[1], expr);
2113
2114         return expr;
2115 }
2116
2117 static int contains_operator(char *str)
2118 {
2119         enum field_op_id field_op = FIELD_OP_NONE;
2120         char *op;
2121
2122         op = strpbrk(str, "+-");
2123         if (!op)
2124                 return FIELD_OP_NONE;
2125
2126         switch (*op) {
2127         case '-':
2128                 if (*str == '-')
2129                         field_op = FIELD_OP_UNARY_MINUS;
2130                 else
2131                         field_op = FIELD_OP_MINUS;
2132                 break;
2133         case '+':
2134                 field_op = FIELD_OP_PLUS;
2135                 break;
2136         default:
2137                 break;
2138         }
2139
2140         return field_op;
2141 }
2142
2143 static void destroy_hist_field(struct hist_field *hist_field,
2144                                unsigned int level)
2145 {
2146         unsigned int i;
2147
2148         if (level > 3)
2149                 return;
2150
2151         if (!hist_field)
2152                 return;
2153
2154         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2155                 destroy_hist_field(hist_field->operands[i], level + 1);
2156
2157         kfree(hist_field->var.name);
2158         kfree(hist_field->name);
2159         kfree(hist_field->type);
2160
2161         kfree(hist_field);
2162 }
2163
2164 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2165                                             struct ftrace_event_field *field,
2166                                             unsigned long flags,
2167                                             char *var_name)
2168 {
2169         struct hist_field *hist_field;
2170
2171         if (field && is_function_field(field))
2172                 return NULL;
2173
2174         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2175         if (!hist_field)
2176                 return NULL;
2177
2178         hist_field->hist_data = hist_data;
2179
2180         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2181                 goto out; /* caller will populate */
2182
2183         if (flags & HIST_FIELD_FL_VAR_REF) {
2184                 hist_field->fn = hist_field_var_ref;
2185                 goto out;
2186         }
2187
2188         if (flags & HIST_FIELD_FL_HITCOUNT) {
2189                 hist_field->fn = hist_field_counter;
2190                 hist_field->size = sizeof(u64);
2191                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2192                 if (!hist_field->type)
2193                         goto free;
2194                 goto out;
2195         }
2196
2197         if (flags & HIST_FIELD_FL_STACKTRACE) {
2198                 hist_field->fn = hist_field_none;
2199                 goto out;
2200         }
2201
2202         if (flags & HIST_FIELD_FL_LOG2) {
2203                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2204                 hist_field->fn = hist_field_log2;
2205                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2206                 hist_field->size = hist_field->operands[0]->size;
2207                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2208                 if (!hist_field->type)
2209                         goto free;
2210                 goto out;
2211         }
2212
2213         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2214                 hist_field->fn = hist_field_timestamp;
2215                 hist_field->size = sizeof(u64);
2216                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2217                 if (!hist_field->type)
2218                         goto free;
2219                 goto out;
2220         }
2221
2222         if (flags & HIST_FIELD_FL_CPU) {
2223                 hist_field->fn = hist_field_cpu;
2224                 hist_field->size = sizeof(int);
2225                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2226                 if (!hist_field->type)
2227                         goto free;
2228                 goto out;
2229         }
2230
2231         if (WARN_ON_ONCE(!field))
2232                 goto out;
2233
2234         if (is_string_field(field)) {
2235                 flags |= HIST_FIELD_FL_STRING;
2236
2237                 hist_field->size = MAX_FILTER_STR_VAL;
2238                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2239                 if (!hist_field->type)
2240                         goto free;
2241
2242                 if (field->filter_type == FILTER_STATIC_STRING)
2243                         hist_field->fn = hist_field_string;
2244                 else if (field->filter_type == FILTER_DYN_STRING)
2245                         hist_field->fn = hist_field_dynstring;
2246                 else
2247                         hist_field->fn = hist_field_pstring;
2248         } else {
2249                 hist_field->size = field->size;
2250                 hist_field->is_signed = field->is_signed;
2251                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2252                 if (!hist_field->type)
2253                         goto free;
2254
2255                 hist_field->fn = select_value_fn(field->size,
2256                                                  field->is_signed);
2257                 if (!hist_field->fn) {
2258                         destroy_hist_field(hist_field, 0);
2259                         return NULL;
2260                 }
2261         }
2262  out:
2263         hist_field->field = field;
2264         hist_field->flags = flags;
2265
2266         if (var_name) {
2267                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2268                 if (!hist_field->var.name)
2269                         goto free;
2270         }
2271
2272         return hist_field;
2273  free:
2274         destroy_hist_field(hist_field, 0);
2275         return NULL;
2276 }
2277
2278 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2279 {
2280         unsigned int i;
2281
2282         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2283                 if (hist_data->fields[i]) {
2284                         destroy_hist_field(hist_data->fields[i], 0);
2285                         hist_data->fields[i] = NULL;
2286                 }
2287         }
2288 }
2289
2290 static int init_var_ref(struct hist_field *ref_field,
2291                         struct hist_field *var_field,
2292                         char *system, char *event_name)
2293 {
2294         int err = 0;
2295
2296         ref_field->var.idx = var_field->var.idx;
2297         ref_field->var.hist_data = var_field->hist_data;
2298         ref_field->size = var_field->size;
2299         ref_field->is_signed = var_field->is_signed;
2300         ref_field->flags |= var_field->flags &
2301                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2302
2303         if (system) {
2304                 ref_field->system = kstrdup(system, GFP_KERNEL);
2305                 if (!ref_field->system)
2306                         return -ENOMEM;
2307         }
2308
2309         if (event_name) {
2310                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2311                 if (!ref_field->event_name) {
2312                         err = -ENOMEM;
2313                         goto free;
2314                 }
2315         }
2316
2317         if (var_field->var.name) {
2318                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2319                 if (!ref_field->name) {
2320                         err = -ENOMEM;
2321                         goto free;
2322                 }
2323         } else if (var_field->name) {
2324                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2325                 if (!ref_field->name) {
2326                         err = -ENOMEM;
2327                         goto free;
2328                 }
2329         }
2330
2331         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2332         if (!ref_field->type) {
2333                 err = -ENOMEM;
2334                 goto free;
2335         }
2336  out:
2337         return err;
2338  free:
2339         kfree(ref_field->system);
2340         kfree(ref_field->event_name);
2341         kfree(ref_field->name);
2342
2343         goto out;
2344 }
2345
2346 static struct hist_field *create_var_ref(struct hist_field *var_field,
2347                                          char *system, char *event_name)
2348 {
2349         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2350         struct hist_field *ref_field;
2351
2352         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2353         if (ref_field) {
2354                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2355                         destroy_hist_field(ref_field, 0);
2356                         return NULL;
2357                 }
2358         }
2359
2360         return ref_field;
2361 }
2362
2363 static bool is_var_ref(char *var_name)
2364 {
2365         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2366                 return false;
2367
2368         return true;
2369 }
2370
2371 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2372                                  char *var_name)
2373 {
2374         char *name, *field;
2375         unsigned int i;
2376
2377         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2378                 name = hist_data->attrs->var_defs.name[i];
2379
2380                 if (strcmp(var_name, name) == 0) {
2381                         field = hist_data->attrs->var_defs.expr[i];
2382                         if (contains_operator(field) || is_var_ref(field))
2383                                 continue;
2384                         return field;
2385                 }
2386         }
2387
2388         return NULL;
2389 }
2390
2391 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2392                                  char *system, char *event_name,
2393                                  char *var_name)
2394 {
2395         struct trace_event_call *call;
2396
2397         if (system && event_name) {
2398                 call = hist_data->event_file->event_call;
2399
2400                 if (strcmp(system, call->class->system) != 0)
2401                         return NULL;
2402
2403                 if (strcmp(event_name, trace_event_name(call)) != 0)
2404                         return NULL;
2405         }
2406
2407         if (!!system != !!event_name)
2408                 return NULL;
2409
2410         if (!is_var_ref(var_name))
2411                 return NULL;
2412
2413         var_name++;
2414
2415         return field_name_from_var(hist_data, var_name);
2416 }
2417
2418 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2419                                         char *system, char *event_name,
2420                                         char *var_name)
2421 {
2422         struct hist_field *var_field = NULL, *ref_field = NULL;
2423
2424         if (!is_var_ref(var_name))
2425                 return NULL;
2426
2427         var_name++;
2428
2429         var_field = find_event_var(hist_data, system, event_name, var_name);
2430         if (var_field)
2431                 ref_field = create_var_ref(var_field, system, event_name);
2432
2433         if (!ref_field)
2434                 hist_err_event("Couldn't find variable: $",
2435                                system, event_name, var_name);
2436
2437         return ref_field;
2438 }
2439
2440 static struct ftrace_event_field *
2441 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2442             char *field_str, unsigned long *flags)
2443 {
2444         struct ftrace_event_field *field = NULL;
2445         char *field_name, *modifier, *str;
2446
2447         modifier = str = kstrdup(field_str, GFP_KERNEL);
2448         if (!modifier)
2449                 return ERR_PTR(-ENOMEM);
2450
2451         field_name = strsep(&modifier, ".");
2452         if (modifier) {
2453                 if (strcmp(modifier, "hex") == 0)
2454                         *flags |= HIST_FIELD_FL_HEX;
2455                 else if (strcmp(modifier, "sym") == 0)
2456                         *flags |= HIST_FIELD_FL_SYM;
2457                 else if (strcmp(modifier, "sym-offset") == 0)
2458                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2459                 else if ((strcmp(modifier, "execname") == 0) &&
2460                          (strcmp(field_name, "common_pid") == 0))
2461                         *flags |= HIST_FIELD_FL_EXECNAME;
2462                 else if (strcmp(modifier, "syscall") == 0)
2463                         *flags |= HIST_FIELD_FL_SYSCALL;
2464                 else if (strcmp(modifier, "log2") == 0)
2465                         *flags |= HIST_FIELD_FL_LOG2;
2466                 else if (strcmp(modifier, "usecs") == 0)
2467                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2468                 else {
2469                         hist_err("Invalid field modifier: ", modifier);
2470                         field = ERR_PTR(-EINVAL);
2471                         goto out;
2472                 }
2473         }
2474
2475         if (strcmp(field_name, "common_timestamp") == 0) {
2476                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2477                 hist_data->enable_timestamps = true;
2478                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2479                         hist_data->attrs->ts_in_usecs = true;
2480         } else if (strcmp(field_name, "cpu") == 0)
2481                 *flags |= HIST_FIELD_FL_CPU;
2482         else {
2483                 field = trace_find_event_field(file->event_call, field_name);
2484                 if (!field || !field->size) {
2485                         hist_err("Couldn't find field: ", field_name);
2486                         field = ERR_PTR(-EINVAL);
2487                         goto out;
2488                 }
2489         }
2490  out:
2491         kfree(str);
2492
2493         return field;
2494 }
2495
2496 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2497                                        struct hist_field *var_ref,
2498                                        char *var_name)
2499 {
2500         struct hist_field *alias = NULL;
2501         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2502
2503         alias = create_hist_field(hist_data, NULL, flags, var_name);
2504         if (!alias)
2505                 return NULL;
2506
2507         alias->fn = var_ref->fn;
2508         alias->operands[0] = var_ref;
2509
2510         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2511                 destroy_hist_field(alias, 0);
2512                 return NULL;
2513         }
2514
2515         return alias;
2516 }
2517
2518 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2519                                      struct trace_event_file *file, char *str,
2520                                      unsigned long *flags, char *var_name)
2521 {
2522         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2523         struct ftrace_event_field *field = NULL;
2524         struct hist_field *hist_field = NULL;
2525         int ret = 0;
2526
2527         s = strchr(str, '.');
2528         if (s) {
2529                 s = strchr(++s, '.');
2530                 if (s) {
2531                         ref_system = strsep(&str, ".");
2532                         if (!str) {
2533                                 ret = -EINVAL;
2534                                 goto out;
2535                         }
2536                         ref_event = strsep(&str, ".");
2537                         if (!str) {
2538                                 ret = -EINVAL;
2539                                 goto out;
2540                         }
2541                         ref_var = str;
2542                 }
2543         }
2544
2545         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2546         if (!s) {
2547                 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2548                 if (hist_field) {
2549                         hist_data->var_refs[hist_data->n_var_refs] = hist_field;
2550                         hist_field->var_ref_idx = hist_data->n_var_refs++;
2551                         if (var_name) {
2552                                 hist_field = create_alias(hist_data, hist_field, var_name);
2553                                 if (!hist_field) {
2554                                         ret = -ENOMEM;
2555                                         goto out;
2556                                 }
2557                         }
2558                         return hist_field;
2559                 }
2560         } else
2561                 str = s;
2562
2563         field = parse_field(hist_data, file, str, flags);
2564         if (IS_ERR(field)) {
2565                 ret = PTR_ERR(field);
2566                 goto out;
2567         }
2568
2569         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2570         if (!hist_field) {
2571                 ret = -ENOMEM;
2572                 goto out;
2573         }
2574
2575         return hist_field;
2576  out:
2577         return ERR_PTR(ret);
2578 }
2579
2580 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2581                                      struct trace_event_file *file,
2582                                      char *str, unsigned long flags,
2583                                      char *var_name, unsigned int level);
2584
2585 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2586                                       struct trace_event_file *file,
2587                                       char *str, unsigned long flags,
2588                                       char *var_name, unsigned int level)
2589 {
2590         struct hist_field *operand1, *expr = NULL;
2591         unsigned long operand_flags;
2592         int ret = 0;
2593         char *s;
2594
2595         /* we support only -(xxx) i.e. explicit parens required */
2596
2597         if (level > 3) {
2598                 hist_err("Too many subexpressions (3 max): ", str);
2599                 ret = -EINVAL;
2600                 goto free;
2601         }
2602
2603         str++; /* skip leading '-' */
2604
2605         s = strchr(str, '(');
2606         if (s)
2607                 str++;
2608         else {
2609                 ret = -EINVAL;
2610                 goto free;
2611         }
2612
2613         s = strrchr(str, ')');
2614         if (s)
2615                 *s = '\0';
2616         else {
2617                 ret = -EINVAL; /* no closing ')' */
2618                 goto free;
2619         }
2620
2621         flags |= HIST_FIELD_FL_EXPR;
2622         expr = create_hist_field(hist_data, NULL, flags, var_name);
2623         if (!expr) {
2624                 ret = -ENOMEM;
2625                 goto free;
2626         }
2627
2628         operand_flags = 0;
2629         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2630         if (IS_ERR(operand1)) {
2631                 ret = PTR_ERR(operand1);
2632                 goto free;
2633         }
2634
2635         expr->flags |= operand1->flags &
2636                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2637         expr->fn = hist_field_unary_minus;
2638         expr->operands[0] = operand1;
2639         expr->operator = FIELD_OP_UNARY_MINUS;
2640         expr->name = expr_str(expr, 0);
2641         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2642         if (!expr->type) {
2643                 ret = -ENOMEM;
2644                 goto free;
2645         }
2646
2647         return expr;
2648  free:
2649         destroy_hist_field(expr, 0);
2650         return ERR_PTR(ret);
2651 }
2652
2653 static int check_expr_operands(struct hist_field *operand1,
2654                                struct hist_field *operand2)
2655 {
2656         unsigned long operand1_flags = operand1->flags;
2657         unsigned long operand2_flags = operand2->flags;
2658
2659         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2660             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2661                 struct hist_field *var;
2662
2663                 var = find_var_field(operand1->var.hist_data, operand1->name);
2664                 if (!var)
2665                         return -EINVAL;
2666                 operand1_flags = var->flags;
2667         }
2668
2669         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2670             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2671                 struct hist_field *var;
2672
2673                 var = find_var_field(operand2->var.hist_data, operand2->name);
2674                 if (!var)
2675                         return -EINVAL;
2676                 operand2_flags = var->flags;
2677         }
2678
2679         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2680             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2681                 hist_err("Timestamp units in expression don't match", NULL);
2682                 return -EINVAL;
2683         }
2684
2685         return 0;
2686 }
2687
2688 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2689                                      struct trace_event_file *file,
2690                                      char *str, unsigned long flags,
2691                                      char *var_name, unsigned int level)
2692 {
2693         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2694         unsigned long operand_flags;
2695         int field_op, ret = -EINVAL;
2696         char *sep, *operand1_str;
2697
2698         if (level > 3) {
2699                 hist_err("Too many subexpressions (3 max): ", str);
2700                 return ERR_PTR(-EINVAL);
2701         }
2702
2703         field_op = contains_operator(str);
2704
2705         if (field_op == FIELD_OP_NONE)
2706                 return parse_atom(hist_data, file, str, &flags, var_name);
2707
2708         if (field_op == FIELD_OP_UNARY_MINUS)
2709                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2710
2711         switch (field_op) {
2712         case FIELD_OP_MINUS:
2713                 sep = "-";
2714                 break;
2715         case FIELD_OP_PLUS:
2716                 sep = "+";
2717                 break;
2718         default:
2719                 goto free;
2720         }
2721
2722         operand1_str = strsep(&str, sep);
2723         if (!operand1_str || !str)
2724                 goto free;
2725
2726         operand_flags = 0;
2727         operand1 = parse_atom(hist_data, file, operand1_str,
2728                               &operand_flags, NULL);
2729         if (IS_ERR(operand1)) {
2730                 ret = PTR_ERR(operand1);
2731                 operand1 = NULL;
2732                 goto free;
2733         }
2734
2735         /* rest of string could be another expression e.g. b+c in a+b+c */
2736         operand_flags = 0;
2737         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2738         if (IS_ERR(operand2)) {
2739                 ret = PTR_ERR(operand2);
2740                 operand2 = NULL;
2741                 goto free;
2742         }
2743
2744         ret = check_expr_operands(operand1, operand2);
2745         if (ret)
2746                 goto free;
2747
2748         flags |= HIST_FIELD_FL_EXPR;
2749
2750         flags |= operand1->flags &
2751                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2752
2753         expr = create_hist_field(hist_data, NULL, flags, var_name);
2754         if (!expr) {
2755                 ret = -ENOMEM;
2756                 goto free;
2757         }
2758
2759         operand1->read_once = true;
2760         operand2->read_once = true;
2761
2762         expr->operands[0] = operand1;
2763         expr->operands[1] = operand2;
2764         expr->operator = field_op;
2765         expr->name = expr_str(expr, 0);
2766         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2767         if (!expr->type) {
2768                 ret = -ENOMEM;
2769                 goto free;
2770         }
2771
2772         switch (field_op) {
2773         case FIELD_OP_MINUS:
2774                 expr->fn = hist_field_minus;
2775                 break;
2776         case FIELD_OP_PLUS:
2777                 expr->fn = hist_field_plus;
2778                 break;
2779         default:
2780                 ret = -EINVAL;
2781                 goto free;
2782         }
2783
2784         return expr;
2785  free:
2786         destroy_hist_field(operand1, 0);
2787         destroy_hist_field(operand2, 0);
2788         destroy_hist_field(expr, 0);
2789
2790         return ERR_PTR(ret);
2791 }
2792
2793 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2794                                  struct trace_event_file *file)
2795 {
2796         struct event_trigger_data *test;
2797
2798         list_for_each_entry_rcu(test, &file->triggers, list) {
2799                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2800                         if (test->private_data == hist_data)
2801                                 return test->filter_str;
2802                 }
2803         }
2804
2805         return NULL;
2806 }
2807
2808 static struct event_command trigger_hist_cmd;
2809 static int event_hist_trigger_func(struct event_command *cmd_ops,
2810                                    struct trace_event_file *file,
2811                                    char *glob, char *cmd, char *param);
2812
2813 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2814                             struct hist_trigger_data *hist_data,
2815                             unsigned int n_keys)
2816 {
2817         struct hist_field *target_hist_field, *hist_field;
2818         unsigned int n, i, j;
2819
2820         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2821                 return false;
2822
2823         i = hist_data->n_vals;
2824         j = target_hist_data->n_vals;
2825
2826         for (n = 0; n < n_keys; n++) {
2827                 hist_field = hist_data->fields[i + n];
2828                 target_hist_field = target_hist_data->fields[j + n];
2829
2830                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2831                         return false;
2832                 if (hist_field->size != target_hist_field->size)
2833                         return false;
2834                 if (hist_field->is_signed != target_hist_field->is_signed)
2835                         return false;
2836         }
2837
2838         return true;
2839 }
2840
2841 static struct hist_trigger_data *
2842 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2843                      struct trace_event_file *file)
2844 {
2845         struct hist_trigger_data *hist_data;
2846         struct event_trigger_data *test;
2847         unsigned int n_keys;
2848
2849         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2850
2851         list_for_each_entry_rcu(test, &file->triggers, list) {
2852                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2853                         hist_data = test->private_data;
2854
2855                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2856                                 return hist_data;
2857                 }
2858         }
2859
2860         return NULL;
2861 }
2862
2863 static struct trace_event_file *event_file(struct trace_array *tr,
2864                                            char *system, char *event_name)
2865 {
2866         struct trace_event_file *file;
2867
2868         file = __find_event_file(tr, system, event_name);
2869         if (!file)
2870                 return ERR_PTR(-EINVAL);
2871
2872         return file;
2873 }
2874
2875 static struct hist_field *
2876 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2877                          char *system, char *event_name, char *field_name)
2878 {
2879         struct hist_field *event_var;
2880         char *synthetic_name;
2881
2882         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2883         if (!synthetic_name)
2884                 return ERR_PTR(-ENOMEM);
2885
2886         strcpy(synthetic_name, "synthetic_");
2887         strcat(synthetic_name, field_name);
2888
2889         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2890
2891         kfree(synthetic_name);
2892
2893         return event_var;
2894 }
2895
2896 /**
2897  * create_field_var_hist - Automatically create a histogram and var for a field
2898  * @target_hist_data: The target hist trigger
2899  * @subsys_name: Optional subsystem name
2900  * @event_name: Optional event name
2901  * @field_name: The name of the field (and the resulting variable)
2902  *
2903  * Hist trigger actions fetch data from variables, not directly from
2904  * events.  However, for convenience, users are allowed to directly
2905  * specify an event field in an action, which will be automatically
2906  * converted into a variable on their behalf.
2907
2908  * If a user specifies a field on an event that isn't the event the
2909  * histogram currently being defined (the target event histogram), the
2910  * only way that can be accomplished is if a new hist trigger is
2911  * created and the field variable defined on that.
2912  *
2913  * This function creates a new histogram compatible with the target
2914  * event (meaning a histogram with the same key as the target
2915  * histogram), and creates a variable for the specified field, but
2916  * with 'synthetic_' prepended to the variable name in order to avoid
2917  * collision with normal field variables.
2918  *
2919  * Return: The variable created for the field.
2920  */
2921 static struct hist_field *
2922 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2923                       char *subsys_name, char *event_name, char *field_name)
2924 {
2925         struct trace_array *tr = target_hist_data->event_file->tr;
2926         struct hist_field *event_var = ERR_PTR(-EINVAL);
2927         struct hist_trigger_data *hist_data;
2928         unsigned int i, n, first = true;
2929         struct field_var_hist *var_hist;
2930         struct trace_event_file *file;
2931         struct hist_field *key_field;
2932         char *saved_filter;
2933         char *cmd;
2934         int ret;
2935
2936         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2937                 hist_err_event("onmatch: Too many field variables defined: ",
2938                                subsys_name, event_name, field_name);
2939                 return ERR_PTR(-EINVAL);
2940         }
2941
2942         file = event_file(tr, subsys_name, event_name);
2943
2944         if (IS_ERR(file)) {
2945                 hist_err_event("onmatch: Event file not found: ",
2946                                subsys_name, event_name, field_name);
2947                 ret = PTR_ERR(file);
2948                 return ERR_PTR(ret);
2949         }
2950
2951         /*
2952          * Look for a histogram compatible with target.  We'll use the
2953          * found histogram specification to create a new matching
2954          * histogram with our variable on it.  target_hist_data is not
2955          * yet a registered histogram so we can't use that.
2956          */
2957         hist_data = find_compatible_hist(target_hist_data, file);
2958         if (!hist_data) {
2959                 hist_err_event("onmatch: Matching event histogram not found: ",
2960                                subsys_name, event_name, field_name);
2961                 return ERR_PTR(-EINVAL);
2962         }
2963
2964         /* See if a synthetic field variable has already been created */
2965         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2966                                              event_name, field_name);
2967         if (!IS_ERR_OR_NULL(event_var))
2968                 return event_var;
2969
2970         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2971         if (!var_hist)
2972                 return ERR_PTR(-ENOMEM);
2973
2974         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2975         if (!cmd) {
2976                 kfree(var_hist);
2977                 return ERR_PTR(-ENOMEM);
2978         }
2979
2980         /* Use the same keys as the compatible histogram */
2981         strcat(cmd, "keys=");
2982
2983         for_each_hist_key_field(i, hist_data) {
2984                 key_field = hist_data->fields[i];
2985                 if (!first)
2986                         strcat(cmd, ",");
2987                 strcat(cmd, key_field->field->name);
2988                 first = false;
2989         }
2990
2991         /* Create the synthetic field variable specification */
2992         strcat(cmd, ":synthetic_");
2993         strcat(cmd, field_name);
2994         strcat(cmd, "=");
2995         strcat(cmd, field_name);
2996
2997         /* Use the same filter as the compatible histogram */
2998         saved_filter = find_trigger_filter(hist_data, file);
2999         if (saved_filter) {
3000                 strcat(cmd, " if ");
3001                 strcat(cmd, saved_filter);
3002         }
3003
3004         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3005         if (!var_hist->cmd) {
3006                 kfree(cmd);
3007                 kfree(var_hist);
3008                 return ERR_PTR(-ENOMEM);
3009         }
3010
3011         /* Save the compatible histogram information */
3012         var_hist->hist_data = hist_data;
3013
3014         /* Create the new histogram with our variable */
3015         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3016                                       "", "hist", cmd);
3017         if (ret) {
3018                 kfree(cmd);
3019                 kfree(var_hist->cmd);
3020                 kfree(var_hist);
3021                 hist_err_event("onmatch: Couldn't create histogram for field: ",
3022                                subsys_name, event_name, field_name);
3023                 return ERR_PTR(ret);
3024         }
3025
3026         kfree(cmd);
3027
3028         /* If we can't find the variable, something went wrong */
3029         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3030                                              event_name, field_name);
3031         if (IS_ERR_OR_NULL(event_var)) {
3032                 kfree(var_hist->cmd);
3033                 kfree(var_hist);
3034                 hist_err_event("onmatch: Couldn't find synthetic variable: ",
3035                                subsys_name, event_name, field_name);
3036                 return ERR_PTR(-EINVAL);
3037         }
3038
3039         n = target_hist_data->n_field_var_hists;
3040         target_hist_data->field_var_hists[n] = var_hist;
3041         target_hist_data->n_field_var_hists++;
3042
3043         return event_var;
3044 }
3045
3046 static struct hist_field *
3047 find_target_event_var(struct hist_trigger_data *hist_data,
3048                       char *subsys_name, char *event_name, char *var_name)
3049 {
3050         struct trace_event_file *file = hist_data->event_file;
3051         struct hist_field *hist_field = NULL;
3052
3053         if (subsys_name) {
3054                 struct trace_event_call *call;
3055
3056                 if (!event_name)
3057                         return NULL;
3058
3059                 call = file->event_call;
3060
3061                 if (strcmp(subsys_name, call->class->system) != 0)
3062                         return NULL;
3063
3064                 if (strcmp(event_name, trace_event_name(call)) != 0)
3065                         return NULL;
3066         }
3067
3068         hist_field = find_var_field(hist_data, var_name);
3069
3070         return hist_field;
3071 }
3072
3073 static inline void __update_field_vars(struct tracing_map_elt *elt,
3074                                        struct ring_buffer_event *rbe,
3075                                        void *rec,
3076                                        struct field_var **field_vars,
3077                                        unsigned int n_field_vars,
3078                                        unsigned int field_var_str_start)
3079 {
3080         struct hist_elt_data *elt_data = elt->private_data;
3081         unsigned int i, j, var_idx;
3082         u64 var_val;
3083
3084         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3085                 struct field_var *field_var = field_vars[i];
3086                 struct hist_field *var = field_var->var;
3087                 struct hist_field *val = field_var->val;
3088
3089                 var_val = val->fn(val, elt, rbe, rec);
3090                 var_idx = var->var.idx;
3091
3092                 if (val->flags & HIST_FIELD_FL_STRING) {
3093                         char *str = elt_data->field_var_str[j++];
3094                         char *val_str = (char *)(uintptr_t)var_val;
3095
3096                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3097                         var_val = (u64)(uintptr_t)str;
3098                 }
3099                 tracing_map_set_var(elt, var_idx, var_val);
3100         }
3101 }
3102
3103 static void update_field_vars(struct hist_trigger_data *hist_data,
3104                               struct tracing_map_elt *elt,
3105                               struct ring_buffer_event *rbe,
3106                               void *rec)
3107 {
3108         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3109                             hist_data->n_field_vars, 0);
3110 }
3111
3112 static void update_max_vars(struct hist_trigger_data *hist_data,
3113                             struct tracing_map_elt *elt,
3114                             struct ring_buffer_event *rbe,
3115                             void *rec)
3116 {
3117         __update_field_vars(elt, rbe, rec, hist_data->max_vars,
3118                             hist_data->n_max_vars, hist_data->n_field_var_str);
3119 }
3120
3121 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3122                                      struct trace_event_file *file,
3123                                      char *name, int size, const char *type)
3124 {
3125         struct hist_field *var;
3126         int idx;
3127
3128         if (find_var(hist_data, file, name) && !hist_data->remove) {
3129                 var = ERR_PTR(-EINVAL);
3130                 goto out;
3131         }
3132
3133         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3134         if (!var) {
3135                 var = ERR_PTR(-ENOMEM);
3136                 goto out;
3137         }
3138
3139         idx = tracing_map_add_var(hist_data->map);
3140         if (idx < 0) {
3141                 kfree(var);
3142                 var = ERR_PTR(-EINVAL);
3143                 goto out;
3144         }
3145
3146         var->flags = HIST_FIELD_FL_VAR;
3147         var->var.idx = idx;
3148         var->var.hist_data = var->hist_data = hist_data;
3149         var->size = size;
3150         var->var.name = kstrdup(name, GFP_KERNEL);
3151         var->type = kstrdup(type, GFP_KERNEL);
3152         if (!var->var.name || !var->type) {
3153                 kfree(var->var.name);
3154                 kfree(var->type);
3155                 kfree(var);
3156                 var = ERR_PTR(-ENOMEM);
3157         }
3158  out:
3159         return var;
3160 }
3161
3162 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3163                                           struct trace_event_file *file,
3164                                           char *field_name)
3165 {
3166         struct hist_field *val = NULL, *var = NULL;
3167         unsigned long flags = HIST_FIELD_FL_VAR;
3168         struct field_var *field_var;
3169         int ret = 0;
3170
3171         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3172                 hist_err("Too many field variables defined: ", field_name);
3173                 ret = -EINVAL;
3174                 goto err;
3175         }
3176
3177         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3178         if (IS_ERR(val)) {
3179                 hist_err("Couldn't parse field variable: ", field_name);
3180                 ret = PTR_ERR(val);
3181                 goto err;
3182         }
3183
3184         var = create_var(hist_data, file, field_name, val->size, val->type);
3185         if (IS_ERR(var)) {
3186                 hist_err("Couldn't create or find variable: ", field_name);
3187                 kfree(val);
3188                 ret = PTR_ERR(var);
3189                 goto err;
3190         }
3191
3192         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3193         if (!field_var) {
3194                 kfree(val);
3195                 kfree(var);
3196                 ret =  -ENOMEM;
3197                 goto err;
3198         }
3199
3200         field_var->var = var;
3201         field_var->val = val;
3202  out:
3203         return field_var;
3204  err:
3205         field_var = ERR_PTR(ret);
3206         goto out;
3207 }
3208
3209 /**
3210  * create_target_field_var - Automatically create a variable for a field
3211  * @target_hist_data: The target hist trigger
3212  * @subsys_name: Optional subsystem name
3213  * @event_name: Optional event name
3214  * @var_name: The name of the field (and the resulting variable)
3215  *
3216  * Hist trigger actions fetch data from variables, not directly from
3217  * events.  However, for convenience, users are allowed to directly
3218  * specify an event field in an action, which will be automatically
3219  * converted into a variable on their behalf.
3220
3221  * This function creates a field variable with the name var_name on
3222  * the hist trigger currently being defined on the target event.  If
3223  * subsys_name and event_name are specified, this function simply
3224  * verifies that they do in fact match the target event subsystem and
3225  * event name.
3226  *
3227  * Return: The variable created for the field.
3228  */
3229 static struct field_var *
3230 create_target_field_var(struct hist_trigger_data *target_hist_data,
3231                         char *subsys_name, char *event_name, char *var_name)
3232 {
3233         struct trace_event_file *file = target_hist_data->event_file;
3234
3235         if (subsys_name) {
3236                 struct trace_event_call *call;
3237
3238                 if (!event_name)
3239                         return NULL;
3240
3241                 call = file->event_call;
3242
3243                 if (strcmp(subsys_name, call->class->system) != 0)
3244                         return NULL;
3245
3246                 if (strcmp(event_name, trace_event_name(call)) != 0)
3247                         return NULL;
3248         }
3249
3250         return create_field_var(target_hist_data, file, var_name);
3251 }
3252
3253 static void onmax_print(struct seq_file *m,
3254                         struct hist_trigger_data *hist_data,
3255                         struct tracing_map_elt *elt,
3256                         struct action_data *data)
3257 {
3258         unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx;
3259
3260         seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx));
3261
3262         for (i = 0; i < hist_data->n_max_vars; i++) {
3263                 struct hist_field *save_val = hist_data->max_vars[i]->val;
3264                 struct hist_field *save_var = hist_data->max_vars[i]->var;
3265                 u64 val;
3266
3267                 save_var_idx = save_var->var.idx;
3268
3269                 val = tracing_map_read_var(elt, save_var_idx);
3270
3271                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3272                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3273                                    (char *)(uintptr_t)(val));
3274                 } else
3275                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3276         }
3277 }
3278
3279 static void onmax_save(struct hist_trigger_data *hist_data,
3280                        struct tracing_map_elt *elt, void *rec,
3281                        struct ring_buffer_event *rbe,
3282                        struct action_data *data, u64 *var_ref_vals)
3283 {
3284         unsigned int max_idx = data->onmax.max_var->var.idx;
3285         unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx;
3286
3287         u64 var_val, max_val;
3288
3289         var_val = var_ref_vals[max_var_ref_idx];
3290         max_val = tracing_map_read_var(elt, max_idx);
3291
3292         if (var_val <= max_val)
3293                 return;
3294
3295         tracing_map_set_var(elt, max_idx, var_val);
3296
3297         update_max_vars(hist_data, elt, rbe, rec);
3298 }
3299
3300 static void onmax_destroy(struct action_data *data)
3301 {
3302         unsigned int i;
3303
3304         destroy_hist_field(data->onmax.max_var, 0);
3305         destroy_hist_field(data->onmax.var, 0);
3306
3307         kfree(data->onmax.var_str);
3308         kfree(data->onmax.fn_name);
3309
3310         for (i = 0; i < data->n_params; i++)
3311                 kfree(data->params[i]);
3312
3313         kfree(data);
3314 }
3315
3316 static int onmax_create(struct hist_trigger_data *hist_data,
3317                         struct action_data *data)
3318 {
3319         struct trace_event_file *file = hist_data->event_file;
3320         struct hist_field *var_field, *ref_field, *max_var;
3321         unsigned int var_ref_idx = hist_data->n_var_refs;
3322         struct field_var *field_var;
3323         char *onmax_var_str, *param;
3324         unsigned long flags;
3325         unsigned int i;
3326         int ret = 0;
3327
3328         onmax_var_str = data->onmax.var_str;
3329         if (onmax_var_str[0] != '$') {
3330                 hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str);
3331                 return -EINVAL;
3332         }
3333         onmax_var_str++;
3334
3335         var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str);
3336         if (!var_field) {
3337                 hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str);
3338                 return -EINVAL;
3339         }
3340
3341         flags = HIST_FIELD_FL_VAR_REF;
3342         ref_field = create_hist_field(hist_data, NULL, flags, NULL);
3343         if (!ref_field)
3344                 return -ENOMEM;
3345
3346         if (init_var_ref(ref_field, var_field, NULL, NULL)) {
3347                 destroy_hist_field(ref_field, 0);
3348                 ret = -ENOMEM;
3349                 goto out;
3350         }
3351         hist_data->var_refs[hist_data->n_var_refs] = ref_field;
3352         ref_field->var_ref_idx = hist_data->n_var_refs++;
3353         data->onmax.var = ref_field;
3354
3355         data->fn = onmax_save;
3356         data->onmax.max_var_ref_idx = var_ref_idx;
3357         max_var = create_var(hist_data, file, "max", sizeof(u64), "u64");
3358         if (IS_ERR(max_var)) {
3359                 hist_err("onmax: Couldn't create onmax variable: ", "max");
3360                 ret = PTR_ERR(max_var);
3361                 goto out;
3362         }
3363         data->onmax.max_var = max_var;
3364
3365         for (i = 0; i < data->n_params; i++) {
3366                 param = kstrdup(data->params[i], GFP_KERNEL);
3367                 if (!param) {
3368                         ret = -ENOMEM;
3369                         goto out;
3370                 }
3371
3372                 field_var = create_target_field_var(hist_data, NULL, NULL, param);
3373                 if (IS_ERR(field_var)) {
3374                         hist_err("onmax: Couldn't create field variable: ", param);
3375                         ret = PTR_ERR(field_var);
3376                         kfree(param);
3377                         goto out;
3378                 }
3379
3380                 hist_data->max_vars[hist_data->n_max_vars++] = field_var;
3381                 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3382                         hist_data->n_max_var_str++;
3383
3384                 kfree(param);
3385         }
3386  out:
3387         return ret;
3388 }
3389
3390 static int parse_action_params(char *params, struct action_data *data)
3391 {
3392         char *param, *saved_param;
3393         int ret = 0;
3394
3395         while (params) {
3396                 if (data->n_params >= SYNTH_FIELDS_MAX)
3397                         goto out;
3398
3399                 param = strsep(&params, ",");
3400                 if (!param) {
3401                         ret = -EINVAL;
3402                         goto out;
3403                 }
3404
3405                 param = strstrip(param);
3406                 if (strlen(param) < 2) {
3407                         hist_err("Invalid action param: ", param);
3408                         ret = -EINVAL;
3409                         goto out;
3410                 }
3411
3412                 saved_param = kstrdup(param, GFP_KERNEL);
3413                 if (!saved_param) {
3414                         ret = -ENOMEM;
3415                         goto out;
3416                 }
3417
3418                 data->params[data->n_params++] = saved_param;
3419         }
3420  out:
3421         return ret;
3422 }
3423
3424 static struct action_data *onmax_parse(char *str)
3425 {
3426         char *onmax_fn_name, *onmax_var_str;
3427         struct action_data *data;
3428         int ret = -EINVAL;
3429
3430         data = kzalloc(sizeof(*data), GFP_KERNEL);
3431         if (!data)
3432                 return ERR_PTR(-ENOMEM);
3433
3434         onmax_var_str = strsep(&str, ")");
3435         if (!onmax_var_str || !str) {
3436                 ret = -EINVAL;
3437                 goto free;
3438         }
3439
3440         data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL);
3441         if (!data->onmax.var_str) {
3442                 ret = -ENOMEM;
3443                 goto free;
3444         }
3445
3446         strsep(&str, ".");
3447         if (!str)
3448                 goto free;
3449
3450         onmax_fn_name = strsep(&str, "(");
3451         if (!onmax_fn_name || !str)
3452                 goto free;
3453
3454         if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) {
3455                 char *params = strsep(&str, ")");
3456
3457                 if (!params) {
3458                         ret = -EINVAL;
3459                         goto free;
3460                 }
3461
3462                 ret = parse_action_params(params, data);
3463                 if (ret)
3464                         goto free;
3465         } else
3466                 goto free;
3467
3468         data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL);
3469         if (!data->onmax.fn_name) {
3470                 ret = -ENOMEM;
3471                 goto free;
3472         }
3473  out:
3474         return data;
3475  free:
3476         onmax_destroy(data);
3477         data = ERR_PTR(ret);
3478         goto out;
3479 }
3480
3481 static void onmatch_destroy(struct action_data *data)
3482 {
3483         unsigned int i;
3484
3485         mutex_lock(&synth_event_mutex);
3486
3487         kfree(data->onmatch.match_event);
3488         kfree(data->onmatch.match_event_system);
3489         kfree(data->onmatch.synth_event_name);
3490
3491         for (i = 0; i < data->n_params; i++)
3492                 kfree(data->params[i]);
3493
3494         if (data->onmatch.synth_event)
3495                 data->onmatch.synth_event->ref--;
3496
3497         kfree(data);
3498
3499         mutex_unlock(&synth_event_mutex);
3500 }
3501
3502 static void destroy_field_var(struct field_var *field_var)
3503 {
3504         if (!field_var)
3505                 return;
3506
3507         destroy_hist_field(field_var->var, 0);
3508         destroy_hist_field(field_var->val, 0);
3509
3510         kfree(field_var);
3511 }
3512
3513 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3514 {
3515         unsigned int i;
3516
3517         for (i = 0; i < hist_data->n_field_vars; i++)
3518                 destroy_field_var(hist_data->field_vars[i]);
3519 }
3520
3521 static void save_field_var(struct hist_trigger_data *hist_data,
3522                            struct field_var *field_var)
3523 {
3524         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3525
3526         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3527                 hist_data->n_field_var_str++;
3528 }
3529
3530
3531 static void destroy_synth_var_refs(struct hist_trigger_data *hist_data)
3532 {
3533         unsigned int i;
3534
3535         for (i = 0; i < hist_data->n_synth_var_refs; i++)
3536                 destroy_hist_field(hist_data->synth_var_refs[i], 0);
3537 }
3538
3539 static void save_synth_var_ref(struct hist_trigger_data *hist_data,
3540                          struct hist_field *var_ref)
3541 {
3542         hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref;
3543
3544         hist_data->var_refs[hist_data->n_var_refs] = var_ref;
3545         var_ref->var_ref_idx = hist_data->n_var_refs++;
3546 }
3547
3548 static int check_synth_field(struct synth_event *event,
3549                              struct hist_field *hist_field,
3550                              unsigned int field_pos)
3551 {
3552         struct synth_field *field;
3553
3554         if (field_pos >= event->n_fields)
3555                 return -EINVAL;
3556
3557         field = event->fields[field_pos];
3558
3559         if (strcmp(field->type, hist_field->type) != 0)
3560                 return -EINVAL;
3561
3562         return 0;
3563 }
3564
3565 static struct hist_field *
3566 onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data,
3567                  char *system, char *event, char *var)
3568 {
3569         struct hist_field *hist_field;
3570
3571         var++; /* skip '$' */
3572
3573         hist_field = find_target_event_var(hist_data, system, event, var);
3574         if (!hist_field) {
3575                 if (!system) {
3576                         system = data->onmatch.match_event_system;
3577                         event = data->onmatch.match_event;
3578                 }
3579
3580                 hist_field = find_event_var(hist_data, system, event, var);
3581         }
3582
3583         if (!hist_field)
3584                 hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var);
3585
3586         return hist_field;
3587 }
3588
3589 static struct hist_field *
3590 onmatch_create_field_var(struct hist_trigger_data *hist_data,
3591                          struct action_data *data, char *system,
3592                          char *event, char *var)
3593 {
3594         struct hist_field *hist_field = NULL;
3595         struct field_var *field_var;
3596
3597         /*
3598          * First try to create a field var on the target event (the
3599          * currently being defined).  This will create a variable for
3600          * unqualified fields on the target event, or if qualified,
3601          * target fields that have qualified names matching the target.
3602          */
3603         field_var = create_target_field_var(hist_data, system, event, var);
3604
3605         if (field_var && !IS_ERR(field_var)) {
3606                 save_field_var(hist_data, field_var);
3607                 hist_field = field_var->var;
3608         } else {
3609                 field_var = NULL;
3610                 /*
3611                  * If no explicit system.event is specfied, default to
3612                  * looking for fields on the onmatch(system.event.xxx)
3613                  * event.
3614                  */
3615                 if (!system) {
3616                         system = data->onmatch.match_event_system;
3617                         event = data->onmatch.match_event;
3618                 }
3619
3620                 /*
3621                  * At this point, we're looking at a field on another
3622                  * event.  Because we can't modify a hist trigger on
3623                  * another event to add a variable for a field, we need
3624                  * to create a new trigger on that event and create the
3625                  * variable at the same time.
3626                  */
3627                 hist_field = create_field_var_hist(hist_data, system, event, var);
3628                 if (IS_ERR(hist_field))
3629                         goto free;
3630         }
3631  out:
3632         return hist_field;
3633  free:
3634         destroy_field_var(field_var);
3635         hist_field = NULL;
3636         goto out;
3637 }
3638
3639 static int onmatch_create(struct hist_trigger_data *hist_data,
3640                           struct trace_event_file *file,
3641                           struct action_data *data)
3642 {
3643         char *event_name, *param, *system = NULL;
3644         struct hist_field *hist_field, *var_ref;
3645         unsigned int i, var_ref_idx;
3646         unsigned int field_pos = 0;
3647         struct synth_event *event;
3648         int ret = 0;
3649
3650         mutex_lock(&synth_event_mutex);
3651         event = find_synth_event(data->onmatch.synth_event_name);
3652         if (!event) {
3653                 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name);
3654                 mutex_unlock(&synth_event_mutex);
3655                 return -EINVAL;
3656         }
3657         event->ref++;
3658         mutex_unlock(&synth_event_mutex);
3659
3660         var_ref_idx = hist_data->n_var_refs;
3661
3662         for (i = 0; i < data->n_params; i++) {
3663                 char *p;
3664
3665                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3666                 if (!param) {
3667                         ret = -ENOMEM;
3668                         goto err;
3669                 }
3670
3671                 system = strsep(&param, ".");
3672                 if (!param) {
3673                         param = (char *)system;
3674                         system = event_name = NULL;
3675                 } else {
3676                         event_name = strsep(&param, ".");
3677                         if (!param) {
3678                                 kfree(p);
3679                                 ret = -EINVAL;
3680                                 goto err;
3681                         }
3682                 }
3683
3684                 if (param[0] == '$')
3685                         hist_field = onmatch_find_var(hist_data, data, system,
3686                                                       event_name, param);
3687                 else
3688                         hist_field = onmatch_create_field_var(hist_data, data,
3689                                                               system,
3690                                                               event_name,
3691                                                               param);
3692
3693                 if (!hist_field) {
3694                         kfree(p);
3695                         ret = -EINVAL;
3696                         goto err;
3697                 }
3698
3699                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3700                         var_ref = create_var_ref(hist_field, system, event_name);
3701                         if (!var_ref) {
3702                                 kfree(p);
3703                                 ret = -ENOMEM;
3704                                 goto err;
3705                         }
3706
3707                         save_synth_var_ref(hist_data, var_ref);
3708                         field_pos++;
3709                         kfree(p);
3710                         continue;
3711                 }
3712
3713                 hist_err_event("onmatch: Param type doesn't match synthetic event field type: ",
3714                                system, event_name, param);
3715                 kfree(p);
3716                 ret = -EINVAL;
3717                 goto err;
3718         }
3719
3720         if (field_pos != event->n_fields) {
3721                 hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name);
3722                 ret = -EINVAL;
3723                 goto err;
3724         }
3725
3726         data->fn = action_trace;
3727         data->onmatch.synth_event = event;
3728         data->onmatch.var_ref_idx = var_ref_idx;
3729  out:
3730         return ret;
3731  err:
3732         mutex_lock(&synth_event_mutex);
3733         event->ref--;
3734         mutex_unlock(&synth_event_mutex);
3735
3736         goto out;
3737 }
3738
3739 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3740 {
3741         char *match_event, *match_event_system;
3742         char *synth_event_name, *params;
3743         struct action_data *data;
3744         int ret = -EINVAL;
3745
3746         data = kzalloc(sizeof(*data), GFP_KERNEL);
3747         if (!data)
3748                 return ERR_PTR(-ENOMEM);
3749
3750         match_event = strsep(&str, ")");
3751         if (!match_event || !str) {
3752                 hist_err("onmatch: Missing closing paren: ", match_event);
3753                 goto free;
3754         }
3755
3756         match_event_system = strsep(&match_event, ".");
3757         if (!match_event) {
3758                 hist_err("onmatch: Missing subsystem for match event: ", match_event_system);
3759                 goto free;
3760         }
3761
3762         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3763                 hist_err_event("onmatch: Invalid subsystem or event name: ",
3764                                match_event_system, match_event, NULL);
3765                 goto free;
3766         }
3767
3768         data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL);
3769         if (!data->onmatch.match_event) {
3770                 ret = -ENOMEM;
3771                 goto free;
3772         }
3773
3774         data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL);
3775         if (!data->onmatch.match_event_system) {
3776                 ret = -ENOMEM;
3777                 goto free;
3778         }
3779
3780         strsep(&str, ".");
3781         if (!str) {
3782                 hist_err("onmatch: Missing . after onmatch(): ", str);
3783                 goto free;
3784         }
3785
3786         synth_event_name = strsep(&str, "(");
3787         if (!synth_event_name || !str) {
3788                 hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name);
3789                 goto free;
3790         }
3791
3792         data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL);
3793         if (!data->onmatch.synth_event_name) {
3794                 ret = -ENOMEM;
3795                 goto free;
3796         }
3797
3798         params = strsep(&str, ")");
3799         if (!params || !str || (str && strlen(str))) {
3800                 hist_err("onmatch: Missing closing paramlist paren: ", params);
3801                 goto free;
3802         }
3803
3804         ret = parse_action_params(params, data);
3805         if (ret)
3806                 goto free;
3807  out:
3808         return data;
3809  free:
3810         onmatch_destroy(data);
3811         data = ERR_PTR(ret);
3812         goto out;
3813 }
3814
3815 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3816 {
3817         hist_data->fields[HITCOUNT_IDX] =
3818                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3819         if (!hist_data->fields[HITCOUNT_IDX])
3820                 return -ENOMEM;
3821
3822         hist_data->n_vals++;
3823         hist_data->n_fields++;
3824
3825         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3826                 return -EINVAL;
3827
3828         return 0;
3829 }
3830
3831 static int __create_val_field(struct hist_trigger_data *hist_data,
3832                               unsigned int val_idx,
3833                               struct trace_event_file *file,
3834                               char *var_name, char *field_str,
3835                               unsigned long flags)
3836 {
3837         struct hist_field *hist_field;
3838         int ret = 0;
3839
3840         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3841         if (IS_ERR(hist_field)) {
3842                 ret = PTR_ERR(hist_field);
3843                 goto out;
3844         }
3845
3846         hist_data->fields[val_idx] = hist_field;
3847
3848         ++hist_data->n_vals;
3849         ++hist_data->n_fields;
3850
3851         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3852                 ret = -EINVAL;
3853  out:
3854         return ret;
3855 }
3856
3857 static int create_val_field(struct hist_trigger_data *hist_data,
3858                             unsigned int val_idx,
3859                             struct trace_event_file *file,
3860                             char *field_str)
3861 {
3862         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3863                 return -EINVAL;
3864
3865         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3866 }
3867
3868 static int create_var_field(struct hist_trigger_data *hist_data,
3869                             unsigned int val_idx,
3870                             struct trace_event_file *file,
3871                             char *var_name, char *expr_str)
3872 {
3873         unsigned long flags = 0;
3874
3875         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3876                 return -EINVAL;
3877
3878         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3879                 hist_err("Variable already defined: ", var_name);
3880                 return -EINVAL;
3881         }
3882
3883         flags |= HIST_FIELD_FL_VAR;
3884         hist_data->n_vars++;
3885         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3886                 return -EINVAL;
3887
3888         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3889 }
3890
3891 static int create_val_fields(struct hist_trigger_data *hist_data,
3892                              struct trace_event_file *file)
3893 {
3894         char *fields_str, *field_str;
3895         unsigned int i, j = 1;
3896         int ret;
3897
3898         ret = create_hitcount_val(hist_data);
3899         if (ret)
3900                 goto out;
3901
3902         fields_str = hist_data->attrs->vals_str;
3903         if (!fields_str)
3904                 goto out;
3905
3906         strsep(&fields_str, "=");
3907         if (!fields_str)
3908                 goto out;
3909
3910         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3911                      j < TRACING_MAP_VALS_MAX; i++) {
3912                 field_str = strsep(&fields_str, ",");
3913                 if (!field_str)
3914                         break;
3915
3916                 if (strcmp(field_str, "hitcount") == 0)
3917                         continue;
3918
3919                 ret = create_val_field(hist_data, j++, file, field_str);
3920                 if (ret)
3921                         goto out;
3922         }
3923
3924         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3925                 ret = -EINVAL;
3926  out:
3927         return ret;
3928 }
3929
3930 static int create_key_field(struct hist_trigger_data *hist_data,
3931                             unsigned int key_idx,
3932                             unsigned int key_offset,
3933                             struct trace_event_file *file,
3934                             char *field_str)
3935 {
3936         struct hist_field *hist_field = NULL;
3937
3938         unsigned long flags = 0;
3939         unsigned int key_size;
3940         int ret = 0;
3941
3942         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3943                 return -EINVAL;
3944
3945         flags |= HIST_FIELD_FL_KEY;
3946
3947         if (strcmp(field_str, "stacktrace") == 0) {
3948                 flags |= HIST_FIELD_FL_STACKTRACE;
3949                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3950                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3951         } else {
3952                 hist_field = parse_expr(hist_data, file, field_str, flags,
3953                                         NULL, 0);
3954                 if (IS_ERR(hist_field)) {
3955                         ret = PTR_ERR(hist_field);
3956                         goto out;
3957                 }
3958
3959                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
3960                         hist_err("Using variable references as keys not supported: ", field_str);
3961                         destroy_hist_field(hist_field, 0);
3962                         ret = -EINVAL;
3963                         goto out;
3964                 }
3965
3966                 key_size = hist_field->size;
3967         }
3968
3969         hist_data->fields[key_idx] = hist_field;
3970
3971         key_size = ALIGN(key_size, sizeof(u64));
3972         hist_data->fields[key_idx]->size = key_size;
3973         hist_data->fields[key_idx]->offset = key_offset;
3974
3975         hist_data->key_size += key_size;
3976
3977         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3978                 ret = -EINVAL;
3979                 goto out;
3980         }
3981
3982         hist_data->n_keys++;
3983         hist_data->n_fields++;
3984
3985         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3986                 return -EINVAL;
3987
3988         ret = key_size;
3989  out:
3990         return ret;
3991 }
3992
3993 static int create_key_fields(struct hist_trigger_data *hist_data,
3994                              struct trace_event_file *file)
3995 {
3996         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3997         char *fields_str, *field_str;
3998         int ret = -EINVAL;
3999
4000         fields_str = hist_data->attrs->keys_str;
4001         if (!fields_str)
4002                 goto out;
4003
4004         strsep(&fields_str, "=");
4005         if (!fields_str)
4006                 goto out;
4007
4008         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4009                 field_str = strsep(&fields_str, ",");
4010                 if (!field_str)
4011                         break;
4012                 ret = create_key_field(hist_data, i, key_offset,
4013                                        file, field_str);
4014                 if (ret < 0)
4015                         goto out;
4016                 key_offset += ret;
4017         }
4018         if (fields_str) {
4019                 ret = -EINVAL;
4020                 goto out;
4021         }
4022         ret = 0;
4023  out:
4024         return ret;
4025 }
4026
4027 static int create_var_fields(struct hist_trigger_data *hist_data,
4028                              struct trace_event_file *file)
4029 {
4030         unsigned int i, j = hist_data->n_vals;
4031         int ret = 0;
4032
4033         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4034
4035         for (i = 0; i < n_vars; i++) {
4036                 char *var_name = hist_data->attrs->var_defs.name[i];
4037                 char *expr = hist_data->attrs->var_defs.expr[i];
4038
4039                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4040                 if (ret)
4041                         goto out;
4042         }
4043  out:
4044         return ret;
4045 }
4046
4047 static void free_var_defs(struct hist_trigger_data *hist_data)
4048 {
4049         unsigned int i;
4050
4051         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4052                 kfree(hist_data->attrs->var_defs.name[i]);
4053                 kfree(hist_data->attrs->var_defs.expr[i]);
4054         }
4055
4056         hist_data->attrs->var_defs.n_vars = 0;
4057 }
4058
4059 static int parse_var_defs(struct hist_trigger_data *hist_data)
4060 {
4061         char *s, *str, *var_name, *field_str;
4062         unsigned int i, j, n_vars = 0;
4063         int ret = 0;
4064
4065         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4066                 str = hist_data->attrs->assignment_str[i];
4067                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4068                         field_str = strsep(&str, ",");
4069                         if (!field_str)
4070                                 break;
4071
4072                         var_name = strsep(&field_str, "=");
4073                         if (!var_name || !field_str) {
4074                                 hist_err("Malformed assignment: ", var_name);
4075                                 ret = -EINVAL;
4076                                 goto free;
4077                         }
4078
4079                         if (n_vars == TRACING_MAP_VARS_MAX) {
4080                                 hist_err("Too many variables defined: ", var_name);
4081                                 ret = -EINVAL;
4082                                 goto free;
4083                         }
4084
4085                         s = kstrdup(var_name, GFP_KERNEL);
4086                         if (!s) {
4087                                 ret = -ENOMEM;
4088                                 goto free;
4089                         }
4090                         hist_data->attrs->var_defs.name[n_vars] = s;
4091
4092                         s = kstrdup(field_str, GFP_KERNEL);
4093                         if (!s) {
4094                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4095                                 ret = -ENOMEM;
4096                                 goto free;
4097                         }
4098                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4099
4100                         hist_data->attrs->var_defs.n_vars = n_vars;
4101                 }
4102         }
4103
4104         return ret;
4105  free:
4106         free_var_defs(hist_data);
4107
4108         return ret;
4109 }
4110
4111 static int create_hist_fields(struct hist_trigger_data *hist_data,
4112                               struct trace_event_file *file)
4113 {
4114         int ret;
4115
4116         ret = parse_var_defs(hist_data);
4117         if (ret)
4118                 goto out;
4119
4120         ret = create_val_fields(hist_data, file);
4121         if (ret)
4122                 goto out;
4123
4124         ret = create_var_fields(hist_data, file);
4125         if (ret)
4126                 goto out;
4127
4128         ret = create_key_fields(hist_data, file);
4129         if (ret)
4130                 goto out;
4131  out:
4132         free_var_defs(hist_data);
4133
4134         return ret;
4135 }
4136
4137 static int is_descending(const char *str)
4138 {
4139         if (!str)
4140                 return 0;
4141
4142         if (strcmp(str, "descending") == 0)
4143                 return 1;
4144
4145         if (strcmp(str, "ascending") == 0)
4146                 return 0;
4147
4148         return -EINVAL;
4149 }
4150
4151 static int create_sort_keys(struct hist_trigger_data *hist_data)
4152 {
4153         char *fields_str = hist_data->attrs->sort_key_str;
4154         struct tracing_map_sort_key *sort_key;
4155         int descending, ret = 0;
4156         unsigned int i, j, k;
4157
4158         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4159
4160         if (!fields_str)
4161                 goto out;
4162
4163         strsep(&fields_str, "=");
4164         if (!fields_str) {
4165                 ret = -EINVAL;
4166                 goto out;
4167         }
4168
4169         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4170                 struct hist_field *hist_field;
4171                 char *field_str, *field_name;
4172                 const char *test_name;
4173
4174                 sort_key = &hist_data->sort_keys[i];
4175
4176                 field_str = strsep(&fields_str, ",");
4177                 if (!field_str) {
4178                         if (i == 0)
4179                                 ret = -EINVAL;
4180                         break;
4181                 }
4182
4183                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4184                         ret = -EINVAL;
4185                         break;
4186                 }
4187
4188                 field_name = strsep(&field_str, ".");
4189                 if (!field_name) {
4190                         ret = -EINVAL;
4191                         break;
4192                 }
4193
4194                 if (strcmp(field_name, "hitcount") == 0) {
4195                         descending = is_descending(field_str);
4196                         if (descending < 0) {
4197                                 ret = descending;
4198                                 break;
4199                         }
4200                         sort_key->descending = descending;
4201                         continue;
4202                 }
4203
4204                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4205                         unsigned int idx;
4206
4207                         hist_field = hist_data->fields[j];
4208                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4209                                 continue;
4210
4211                         idx = k++;
4212
4213                         test_name = hist_field_name(hist_field, 0);
4214
4215                         if (strcmp(field_name, test_name) == 0) {
4216                                 sort_key->field_idx = idx;
4217                                 descending = is_descending(field_str);
4218                                 if (descending < 0) {
4219                                         ret = descending;
4220                                         goto out;
4221                                 }
4222                                 sort_key->descending = descending;
4223                                 break;
4224                         }
4225                 }
4226                 if (j == hist_data->n_fields) {
4227                         ret = -EINVAL;
4228                         break;
4229                 }
4230         }
4231
4232         hist_data->n_sort_keys = i;
4233  out:
4234         return ret;
4235 }
4236
4237 static void destroy_actions(struct hist_trigger_data *hist_data)
4238 {
4239         unsigned int i;
4240
4241         for (i = 0; i < hist_data->n_actions; i++) {
4242                 struct action_data *data = hist_data->actions[i];
4243
4244                 if (data->fn == action_trace)
4245                         onmatch_destroy(data);
4246                 else if (data->fn == onmax_save)
4247                         onmax_destroy(data);
4248                 else
4249                         kfree(data);
4250         }
4251 }
4252
4253 static int parse_actions(struct hist_trigger_data *hist_data)
4254 {
4255         struct trace_array *tr = hist_data->event_file->tr;
4256         struct action_data *data;
4257         unsigned int i;
4258         int ret = 0;
4259         char *str;
4260
4261         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4262                 str = hist_data->attrs->action_str[i];
4263
4264                 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) {
4265                         char *action_str = str + strlen("onmatch(");
4266
4267                         data = onmatch_parse(tr, action_str);
4268                         if (IS_ERR(data)) {
4269                                 ret = PTR_ERR(data);
4270                                 break;
4271                         }
4272                         data->fn = action_trace;
4273                 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) {
4274                         char *action_str = str + strlen("onmax(");
4275
4276                         data = onmax_parse(action_str);
4277                         if (IS_ERR(data)) {
4278                                 ret = PTR_ERR(data);
4279                                 break;
4280                         }
4281                         data->fn = onmax_save;
4282                 } else {
4283                         ret = -EINVAL;
4284                         break;
4285                 }
4286
4287                 hist_data->actions[hist_data->n_actions++] = data;
4288         }
4289
4290         return ret;
4291 }
4292
4293 static int create_actions(struct hist_trigger_data *hist_data,
4294                           struct trace_event_file *file)
4295 {
4296         struct action_data *data;
4297         unsigned int i;
4298         int ret = 0;
4299
4300         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4301                 data = hist_data->actions[i];
4302
4303                 if (data->fn == action_trace) {
4304                         ret = onmatch_create(hist_data, file, data);
4305                         if (ret)
4306                                 return ret;
4307                 } else if (data->fn == onmax_save) {
4308                         ret = onmax_create(hist_data, data);
4309                         if (ret)
4310                                 return ret;
4311                 }
4312         }
4313
4314         return ret;
4315 }
4316
4317 static void print_actions(struct seq_file *m,
4318                           struct hist_trigger_data *hist_data,
4319                           struct tracing_map_elt *elt)
4320 {
4321         unsigned int i;
4322
4323         for (i = 0; i < hist_data->n_actions; i++) {
4324                 struct action_data *data = hist_data->actions[i];
4325
4326                 if (data->fn == onmax_save)
4327                         onmax_print(m, hist_data, elt, data);
4328         }
4329 }
4330
4331 static void print_onmax_spec(struct seq_file *m,
4332                              struct hist_trigger_data *hist_data,
4333                              struct action_data *data)
4334 {
4335         unsigned int i;
4336
4337         seq_puts(m, ":onmax(");
4338         seq_printf(m, "%s", data->onmax.var_str);
4339         seq_printf(m, ").%s(", data->onmax.fn_name);
4340
4341         for (i = 0; i < hist_data->n_max_vars; i++) {
4342                 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name);
4343                 if (i < hist_data->n_max_vars - 1)
4344                         seq_puts(m, ",");
4345         }
4346         seq_puts(m, ")");
4347 }
4348
4349 static void print_onmatch_spec(struct seq_file *m,
4350                                struct hist_trigger_data *hist_data,
4351                                struct action_data *data)
4352 {
4353         unsigned int i;
4354
4355         seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system,
4356                    data->onmatch.match_event);
4357
4358         seq_printf(m, "%s(", data->onmatch.synth_event->name);
4359
4360         for (i = 0; i < data->n_params; i++) {
4361                 if (i)
4362                         seq_puts(m, ",");
4363                 seq_printf(m, "%s", data->params[i]);
4364         }
4365
4366         seq_puts(m, ")");
4367 }
4368
4369 static bool actions_match(struct hist_trigger_data *hist_data,
4370                           struct hist_trigger_data *hist_data_test)
4371 {
4372         unsigned int i, j;
4373
4374         if (hist_data->n_actions != hist_data_test->n_actions)
4375                 return false;
4376
4377         for (i = 0; i < hist_data->n_actions; i++) {
4378                 struct action_data *data = hist_data->actions[i];
4379                 struct action_data *data_test = hist_data_test->actions[i];
4380
4381                 if (data->fn != data_test->fn)
4382                         return false;
4383
4384                 if (data->n_params != data_test->n_params)
4385                         return false;
4386
4387                 for (j = 0; j < data->n_params; j++) {
4388                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4389                                 return false;
4390                 }
4391
4392                 if (data->fn == action_trace) {
4393                         if (strcmp(data->onmatch.synth_event_name,
4394                                    data_test->onmatch.synth_event_name) != 0)
4395                                 return false;
4396                         if (strcmp(data->onmatch.match_event_system,
4397                                    data_test->onmatch.match_event_system) != 0)
4398                                 return false;
4399                         if (strcmp(data->onmatch.match_event,
4400                                    data_test->onmatch.match_event) != 0)
4401                                 return false;
4402                 } else if (data->fn == onmax_save) {
4403                         if (strcmp(data->onmax.var_str,
4404                                    data_test->onmax.var_str) != 0)
4405                                 return false;
4406                         if (strcmp(data->onmax.fn_name,
4407                                    data_test->onmax.fn_name) != 0)
4408                                 return false;
4409                 }
4410         }
4411
4412         return true;
4413 }
4414
4415
4416 static void print_actions_spec(struct seq_file *m,
4417                                struct hist_trigger_data *hist_data)
4418 {
4419         unsigned int i;
4420
4421         for (i = 0; i < hist_data->n_actions; i++) {
4422                 struct action_data *data = hist_data->actions[i];
4423
4424                 if (data->fn == action_trace)
4425                         print_onmatch_spec(m, hist_data, data);
4426                 else if (data->fn == onmax_save)
4427                         print_onmax_spec(m, hist_data, data);
4428         }
4429 }
4430
4431 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4432 {
4433         unsigned int i;
4434
4435         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4436                 kfree(hist_data->field_var_hists[i]->cmd);
4437                 kfree(hist_data->field_var_hists[i]);
4438         }
4439 }
4440
4441 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4442 {
4443         if (!hist_data)
4444                 return;
4445
4446         destroy_hist_trigger_attrs(hist_data->attrs);
4447         destroy_hist_fields(hist_data);
4448         tracing_map_destroy(hist_data->map);
4449
4450         destroy_actions(hist_data);
4451         destroy_field_vars(hist_data);
4452         destroy_field_var_hists(hist_data);
4453         destroy_synth_var_refs(hist_data);
4454
4455         kfree(hist_data);
4456 }
4457
4458 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4459 {
4460         struct tracing_map *map = hist_data->map;
4461         struct ftrace_event_field *field;
4462         struct hist_field *hist_field;
4463         int i, idx = 0;
4464
4465         for_each_hist_field(i, hist_data) {
4466                 hist_field = hist_data->fields[i];
4467                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4468                         tracing_map_cmp_fn_t cmp_fn;
4469
4470                         field = hist_field->field;
4471
4472                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4473                                 cmp_fn = tracing_map_cmp_none;
4474                         else if (!field)
4475                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4476                                                              hist_field->is_signed);
4477                         else if (is_string_field(field))
4478                                 cmp_fn = tracing_map_cmp_string;
4479                         else
4480                                 cmp_fn = tracing_map_cmp_num(field->size,
4481                                                              field->is_signed);
4482                         idx = tracing_map_add_key_field(map,
4483                                                         hist_field->offset,
4484                                                         cmp_fn);
4485                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4486                         idx = tracing_map_add_sum_field(map);
4487
4488                 if (idx < 0)
4489                         return idx;
4490
4491                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4492                         idx = tracing_map_add_var(map);
4493                         if (idx < 0)
4494                                 return idx;
4495                         hist_field->var.idx = idx;
4496                         hist_field->var.hist_data = hist_data;
4497                 }
4498         }
4499
4500         return 0;
4501 }
4502
4503 static struct hist_trigger_data *
4504 create_hist_data(unsigned int map_bits,
4505                  struct hist_trigger_attrs *attrs,
4506                  struct trace_event_file *file,
4507                  bool remove)
4508 {
4509         const struct tracing_map_ops *map_ops = NULL;
4510         struct hist_trigger_data *hist_data;
4511         int ret = 0;
4512
4513         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4514         if (!hist_data)
4515                 return ERR_PTR(-ENOMEM);
4516
4517         hist_data->attrs = attrs;
4518         hist_data->remove = remove;
4519         hist_data->event_file = file;
4520
4521         ret = parse_actions(hist_data);
4522         if (ret)
4523                 goto free;
4524
4525         ret = create_hist_fields(hist_data, file);
4526         if (ret)
4527                 goto free;
4528
4529         ret = create_sort_keys(hist_data);
4530         if (ret)
4531                 goto free;
4532
4533         map_ops = &hist_trigger_elt_data_ops;
4534
4535         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4536                                             map_ops, hist_data);
4537         if (IS_ERR(hist_data->map)) {
4538                 ret = PTR_ERR(hist_data->map);
4539                 hist_data->map = NULL;
4540                 goto free;
4541         }
4542
4543         ret = create_tracing_map_fields(hist_data);
4544         if (ret)
4545                 goto free;
4546  out:
4547         return hist_data;
4548  free:
4549         hist_data->attrs = NULL;
4550
4551         destroy_hist_data(hist_data);
4552
4553         hist_data = ERR_PTR(ret);
4554
4555         goto out;
4556 }
4557
4558 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4559                                     struct tracing_map_elt *elt, void *rec,
4560                                     struct ring_buffer_event *rbe,
4561                                     u64 *var_ref_vals)
4562 {
4563         struct hist_elt_data *elt_data;
4564         struct hist_field *hist_field;
4565         unsigned int i, var_idx;
4566         u64 hist_val;
4567
4568         elt_data = elt->private_data;
4569         elt_data->var_ref_vals = var_ref_vals;
4570
4571         for_each_hist_val_field(i, hist_data) {
4572                 hist_field = hist_data->fields[i];
4573                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4574                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4575                         var_idx = hist_field->var.idx;
4576                         tracing_map_set_var(elt, var_idx, hist_val);
4577                         continue;
4578                 }
4579                 tracing_map_update_sum(elt, i, hist_val);
4580         }
4581
4582         for_each_hist_key_field(i, hist_data) {
4583                 hist_field = hist_data->fields[i];
4584                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4585                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4586                         var_idx = hist_field->var.idx;
4587                         tracing_map_set_var(elt, var_idx, hist_val);
4588                 }
4589         }
4590
4591         update_field_vars(hist_data, elt, rbe, rec);
4592 }
4593
4594 static inline void add_to_key(char *compound_key, void *key,
4595                               struct hist_field *key_field, void *rec)
4596 {
4597         size_t size = key_field->size;
4598
4599         if (key_field->flags & HIST_FIELD_FL_STRING) {
4600                 struct ftrace_event_field *field;
4601
4602                 field = key_field->field;
4603                 if (field->filter_type == FILTER_DYN_STRING)
4604                         size = *(u32 *)(rec + field->offset) >> 16;
4605                 else if (field->filter_type == FILTER_PTR_STRING)
4606                         size = strlen(key);
4607                 else if (field->filter_type == FILTER_STATIC_STRING)
4608                         size = field->size;
4609
4610                 /* ensure NULL-termination */
4611                 if (size > key_field->size - 1)
4612                         size = key_field->size - 1;
4613         }
4614
4615         memcpy(compound_key + key_field->offset, key, size);
4616 }
4617
4618 static void
4619 hist_trigger_actions(struct hist_trigger_data *hist_data,
4620                      struct tracing_map_elt *elt, void *rec,
4621                      struct ring_buffer_event *rbe, u64 *var_ref_vals)
4622 {
4623         struct action_data *data;
4624         unsigned int i;
4625
4626         for (i = 0; i < hist_data->n_actions; i++) {
4627                 data = hist_data->actions[i];
4628                 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals);
4629         }
4630 }
4631
4632 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4633                                struct ring_buffer_event *rbe)
4634 {
4635         struct hist_trigger_data *hist_data = data->private_data;
4636         bool use_compound_key = (hist_data->n_keys > 1);
4637         unsigned long entries[HIST_STACKTRACE_DEPTH];
4638         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4639         char compound_key[HIST_KEY_SIZE_MAX];
4640         struct tracing_map_elt *elt = NULL;
4641         struct stack_trace stacktrace;
4642         struct hist_field *key_field;
4643         u64 field_contents;
4644         void *key = NULL;
4645         unsigned int i;
4646
4647         memset(compound_key, 0, hist_data->key_size);
4648
4649         for_each_hist_key_field(i, hist_data) {
4650                 key_field = hist_data->fields[i];
4651
4652                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4653                         stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
4654                         stacktrace.entries = entries;
4655                         stacktrace.nr_entries = 0;
4656                         stacktrace.skip = HIST_STACKTRACE_SKIP;
4657
4658                         memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
4659                         save_stack_trace(&stacktrace);
4660
4661                         key = entries;
4662                 } else {
4663                         field_contents = key_field->fn(key_field, elt, rbe, rec);
4664                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4665                                 key = (void *)(unsigned long)field_contents;
4666                                 use_compound_key = true;
4667                         } else
4668                                 key = (void *)&field_contents;
4669                 }
4670
4671                 if (use_compound_key)
4672                         add_to_key(compound_key, key, key_field, rec);
4673         }
4674
4675         if (use_compound_key)
4676                 key = compound_key;
4677
4678         if (hist_data->n_var_refs &&
4679             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4680                 return;
4681
4682         elt = tracing_map_insert(hist_data->map, key);
4683         if (!elt)
4684                 return;
4685
4686         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4687
4688         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4689                 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals);
4690 }
4691
4692 static void hist_trigger_stacktrace_print(struct seq_file *m,
4693                                           unsigned long *stacktrace_entries,
4694                                           unsigned int max_entries)
4695 {
4696         char str[KSYM_SYMBOL_LEN];
4697         unsigned int spaces = 8;
4698         unsigned int i;
4699
4700         for (i = 0; i < max_entries; i++) {
4701                 if (stacktrace_entries[i] == ULONG_MAX)
4702                         return;
4703
4704                 seq_printf(m, "%*c", 1 + spaces, ' ');
4705                 sprint_symbol(str, stacktrace_entries[i]);
4706                 seq_printf(m, "%s\n", str);
4707         }
4708 }
4709
4710 static void
4711 hist_trigger_entry_print(struct seq_file *m,
4712                          struct hist_trigger_data *hist_data, void *key,
4713                          struct tracing_map_elt *elt)
4714 {
4715         struct hist_field *key_field;
4716         char str[KSYM_SYMBOL_LEN];
4717         bool multiline = false;
4718         const char *field_name;
4719         unsigned int i;
4720         u64 uval;
4721
4722         seq_puts(m, "{ ");
4723
4724         for_each_hist_key_field(i, hist_data) {
4725                 key_field = hist_data->fields[i];
4726
4727                 if (i > hist_data->n_vals)
4728                         seq_puts(m, ", ");
4729
4730                 field_name = hist_field_name(key_field, 0);
4731
4732                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4733                         uval = *(u64 *)(key + key_field->offset);
4734                         seq_printf(m, "%s: %llx", field_name, uval);
4735                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4736                         uval = *(u64 *)(key + key_field->offset);
4737                         sprint_symbol_no_offset(str, uval);
4738                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4739                                    uval, str);
4740                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4741                         uval = *(u64 *)(key + key_field->offset);
4742                         sprint_symbol(str, uval);
4743                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4744                                    uval, str);
4745                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4746                         struct hist_elt_data *elt_data = elt->private_data;
4747                         char *comm;
4748
4749                         if (WARN_ON_ONCE(!elt_data))
4750                                 return;
4751
4752                         comm = elt_data->comm;
4753
4754                         uval = *(u64 *)(key + key_field->offset);
4755                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4756                                    comm, uval);
4757                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4758                         const char *syscall_name;
4759
4760                         uval = *(u64 *)(key + key_field->offset);
4761                         syscall_name = get_syscall_name(uval);
4762                         if (!syscall_name)
4763                                 syscall_name = "unknown_syscall";
4764
4765                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4766                                    syscall_name, uval);
4767                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4768                         seq_puts(m, "stacktrace:\n");
4769                         hist_trigger_stacktrace_print(m,
4770                                                       key + key_field->offset,
4771                                                       HIST_STACKTRACE_DEPTH);
4772                         multiline = true;
4773                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4774                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4775                                    *(u64 *)(key + key_field->offset));
4776                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4777                         seq_printf(m, "%s: %-50s", field_name,
4778                                    (char *)(key + key_field->offset));
4779                 } else {
4780                         uval = *(u64 *)(key + key_field->offset);
4781                         seq_printf(m, "%s: %10llu", field_name, uval);
4782                 }
4783         }
4784
4785         if (!multiline)
4786                 seq_puts(m, " ");
4787
4788         seq_puts(m, "}");
4789
4790         seq_printf(m, " hitcount: %10llu",
4791                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4792
4793         for (i = 1; i < hist_data->n_vals; i++) {
4794                 field_name = hist_field_name(hist_data->fields[i], 0);
4795
4796                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4797                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4798                         continue;
4799
4800                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4801                         seq_printf(m, "  %s: %10llx", field_name,
4802                                    tracing_map_read_sum(elt, i));
4803                 } else {
4804                         seq_printf(m, "  %s: %10llu", field_name,
4805                                    tracing_map_read_sum(elt, i));
4806                 }
4807         }
4808
4809         print_actions(m, hist_data, elt);
4810
4811         seq_puts(m, "\n");
4812 }
4813
4814 static int print_entries(struct seq_file *m,
4815                          struct hist_trigger_data *hist_data)
4816 {
4817         struct tracing_map_sort_entry **sort_entries = NULL;
4818         struct tracing_map *map = hist_data->map;
4819         int i, n_entries;
4820
4821         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4822                                              hist_data->n_sort_keys,
4823                                              &sort_entries);
4824         if (n_entries < 0)
4825                 return n_entries;
4826
4827         for (i = 0; i < n_entries; i++)
4828                 hist_trigger_entry_print(m, hist_data,
4829                                          sort_entries[i]->key,
4830                                          sort_entries[i]->elt);
4831
4832         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4833
4834         return n_entries;
4835 }
4836
4837 static void hist_trigger_show(struct seq_file *m,
4838                               struct event_trigger_data *data, int n)
4839 {
4840         struct hist_trigger_data *hist_data;
4841         int n_entries;
4842
4843         if (n > 0)
4844                 seq_puts(m, "\n\n");
4845
4846         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4847         data->ops->print(m, data->ops, data);
4848         seq_puts(m, "#\n\n");
4849
4850         hist_data = data->private_data;
4851         n_entries = print_entries(m, hist_data);
4852         if (n_entries < 0)
4853                 n_entries = 0;
4854
4855         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4856                    (u64)atomic64_read(&hist_data->map->hits),
4857                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4858 }
4859
4860 static int hist_show(struct seq_file *m, void *v)
4861 {
4862         struct event_trigger_data *data;
4863         struct trace_event_file *event_file;
4864         int n = 0, ret = 0;
4865
4866         mutex_lock(&event_mutex);
4867
4868         event_file = event_file_data(m->private);
4869         if (unlikely(!event_file)) {
4870                 ret = -ENODEV;
4871                 goto out_unlock;
4872         }
4873
4874         list_for_each_entry_rcu(data, &event_file->triggers, list) {
4875                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4876                         hist_trigger_show(m, data, n++);
4877         }
4878
4879         if (have_hist_err()) {
4880                 seq_printf(m, "\nERROR: %s\n", hist_err_str);
4881                 seq_printf(m, "  Last command: %s\n", last_hist_cmd);
4882         }
4883
4884  out_unlock:
4885         mutex_unlock(&event_mutex);
4886
4887         return ret;
4888 }
4889
4890 static int event_hist_open(struct inode *inode, struct file *file)
4891 {
4892         return single_open(file, hist_show, file);
4893 }
4894
4895 const struct file_operations event_hist_fops = {
4896         .open = event_hist_open,
4897         .read = seq_read,
4898         .llseek = seq_lseek,
4899         .release = single_release,
4900 };
4901
4902 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
4903 {
4904         const char *field_name = hist_field_name(hist_field, 0);
4905
4906         if (hist_field->var.name)
4907                 seq_printf(m, "%s=", hist_field->var.name);
4908
4909         if (hist_field->flags & HIST_FIELD_FL_CPU)
4910                 seq_puts(m, "cpu");
4911         else if (field_name) {
4912                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
4913                     hist_field->flags & HIST_FIELD_FL_ALIAS)
4914                         seq_putc(m, '$');
4915                 seq_printf(m, "%s", field_name);
4916         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
4917                 seq_puts(m, "common_timestamp");
4918
4919         if (hist_field->flags) {
4920                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
4921                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
4922                         const char *flags = get_hist_field_flags(hist_field);
4923
4924                         if (flags)
4925                                 seq_printf(m, ".%s", flags);
4926                 }
4927         }
4928 }
4929
4930 static int event_hist_trigger_print(struct seq_file *m,
4931                                     struct event_trigger_ops *ops,
4932                                     struct event_trigger_data *data)
4933 {
4934         struct hist_trigger_data *hist_data = data->private_data;
4935         struct hist_field *field;
4936         bool have_var = false;
4937         unsigned int i;
4938
4939         seq_puts(m, "hist:");
4940
4941         if (data->name)
4942                 seq_printf(m, "%s:", data->name);
4943
4944         seq_puts(m, "keys=");
4945
4946         for_each_hist_key_field(i, hist_data) {
4947                 field = hist_data->fields[i];
4948
4949                 if (i > hist_data->n_vals)
4950                         seq_puts(m, ",");
4951
4952                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
4953                         seq_puts(m, "stacktrace");
4954                 else
4955                         hist_field_print(m, field);
4956         }
4957
4958         seq_puts(m, ":vals=");
4959
4960         for_each_hist_val_field(i, hist_data) {
4961                 field = hist_data->fields[i];
4962                 if (field->flags & HIST_FIELD_FL_VAR) {
4963                         have_var = true;
4964                         continue;
4965                 }
4966
4967                 if (i == HITCOUNT_IDX)
4968                         seq_puts(m, "hitcount");
4969                 else {
4970                         seq_puts(m, ",");
4971                         hist_field_print(m, field);
4972                 }
4973         }
4974
4975         if (have_var) {
4976                 unsigned int n = 0;
4977
4978                 seq_puts(m, ":");
4979
4980                 for_each_hist_val_field(i, hist_data) {
4981                         field = hist_data->fields[i];
4982
4983                         if (field->flags & HIST_FIELD_FL_VAR) {
4984                                 if (n++)
4985                                         seq_puts(m, ",");
4986                                 hist_field_print(m, field);
4987                         }
4988                 }
4989         }
4990
4991         seq_puts(m, ":sort=");
4992
4993         for (i = 0; i < hist_data->n_sort_keys; i++) {
4994                 struct tracing_map_sort_key *sort_key;
4995                 unsigned int idx, first_key_idx;
4996
4997                 /* skip VAR vals */
4998                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
4999
5000                 sort_key = &hist_data->sort_keys[i];
5001                 idx = sort_key->field_idx;
5002
5003                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5004                         return -EINVAL;
5005
5006                 if (i > 0)
5007                         seq_puts(m, ",");
5008
5009                 if (idx == HITCOUNT_IDX)
5010                         seq_puts(m, "hitcount");
5011                 else {
5012                         if (idx >= first_key_idx)
5013                                 idx += hist_data->n_vars;
5014                         hist_field_print(m, hist_data->fields[idx]);
5015                 }
5016
5017                 if (sort_key->descending)
5018                         seq_puts(m, ".descending");
5019         }
5020         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5021         if (hist_data->enable_timestamps)
5022                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5023
5024         print_actions_spec(m, hist_data);
5025
5026         if (data->filter_str)
5027                 seq_printf(m, " if %s", data->filter_str);
5028
5029         if (data->paused)
5030                 seq_puts(m, " [paused]");
5031         else
5032                 seq_puts(m, " [active]");
5033
5034         seq_putc(m, '\n');
5035
5036         return 0;
5037 }
5038
5039 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5040                                    struct event_trigger_data *data)
5041 {
5042         struct hist_trigger_data *hist_data = data->private_data;
5043
5044         if (!data->ref && hist_data->attrs->name)
5045                 save_named_trigger(hist_data->attrs->name, data);
5046
5047         data->ref++;
5048
5049         return 0;
5050 }
5051
5052 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5053 {
5054         struct trace_event_file *file;
5055         unsigned int i;
5056         char *cmd;
5057         int ret;
5058
5059         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5060                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5061                 cmd = hist_data->field_var_hists[i]->cmd;
5062                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5063                                               "!hist", "hist", cmd);
5064         }
5065 }
5066
5067 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5068                                     struct event_trigger_data *data)
5069 {
5070         struct hist_trigger_data *hist_data = data->private_data;
5071
5072         if (WARN_ON_ONCE(data->ref <= 0))
5073                 return;
5074
5075         data->ref--;
5076         if (!data->ref) {
5077                 if (data->name)
5078                         del_named_trigger(data);
5079
5080                 trigger_data_free(data);
5081
5082                 remove_hist_vars(hist_data);
5083
5084                 unregister_field_var_hists(hist_data);
5085
5086                 destroy_hist_data(hist_data);
5087         }
5088 }
5089
5090 static struct event_trigger_ops event_hist_trigger_ops = {
5091         .func                   = event_hist_trigger,
5092         .print                  = event_hist_trigger_print,
5093         .init                   = event_hist_trigger_init,
5094         .free                   = event_hist_trigger_free,
5095 };
5096
5097 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5098                                          struct event_trigger_data *data)
5099 {
5100         data->ref++;
5101
5102         save_named_trigger(data->named_data->name, data);
5103
5104         event_hist_trigger_init(ops, data->named_data);
5105
5106         return 0;
5107 }
5108
5109 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5110                                           struct event_trigger_data *data)
5111 {
5112         if (WARN_ON_ONCE(data->ref <= 0))
5113                 return;
5114
5115         event_hist_trigger_free(ops, data->named_data);
5116
5117         data->ref--;
5118         if (!data->ref) {
5119                 del_named_trigger(data);
5120                 trigger_data_free(data);
5121         }
5122 }
5123
5124 static struct event_trigger_ops event_hist_trigger_named_ops = {
5125         .func                   = event_hist_trigger,
5126         .print                  = event_hist_trigger_print,
5127         .init                   = event_hist_trigger_named_init,
5128         .free                   = event_hist_trigger_named_free,
5129 };
5130
5131 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5132                                                             char *param)
5133 {
5134         return &event_hist_trigger_ops;
5135 }
5136
5137 static void hist_clear(struct event_trigger_data *data)
5138 {
5139         struct hist_trigger_data *hist_data = data->private_data;
5140
5141         if (data->name)
5142                 pause_named_trigger(data);
5143
5144         synchronize_sched();
5145
5146         tracing_map_clear(hist_data->map);
5147
5148         if (data->name)
5149                 unpause_named_trigger(data);
5150 }
5151
5152 static bool compatible_field(struct ftrace_event_field *field,
5153                              struct ftrace_event_field *test_field)
5154 {
5155         if (field == test_field)
5156                 return true;
5157         if (field == NULL || test_field == NULL)
5158                 return false;
5159         if (strcmp(field->name, test_field->name) != 0)
5160                 return false;
5161         if (strcmp(field->type, test_field->type) != 0)
5162                 return false;
5163         if (field->size != test_field->size)
5164                 return false;
5165         if (field->is_signed != test_field->is_signed)
5166                 return false;
5167
5168         return true;
5169 }
5170
5171 static bool hist_trigger_match(struct event_trigger_data *data,
5172                                struct event_trigger_data *data_test,
5173                                struct event_trigger_data *named_data,
5174                                bool ignore_filter)
5175 {
5176         struct tracing_map_sort_key *sort_key, *sort_key_test;
5177         struct hist_trigger_data *hist_data, *hist_data_test;
5178         struct hist_field *key_field, *key_field_test;
5179         unsigned int i;
5180
5181         if (named_data && (named_data != data_test) &&
5182             (named_data != data_test->named_data))
5183                 return false;
5184
5185         if (!named_data && is_named_trigger(data_test))
5186                 return false;
5187
5188         hist_data = data->private_data;
5189         hist_data_test = data_test->private_data;
5190
5191         if (hist_data->n_vals != hist_data_test->n_vals ||
5192             hist_data->n_fields != hist_data_test->n_fields ||
5193             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5194                 return false;
5195
5196         if (!ignore_filter) {
5197                 if ((data->filter_str && !data_test->filter_str) ||
5198                    (!data->filter_str && data_test->filter_str))
5199                         return false;
5200         }
5201
5202         for_each_hist_field(i, hist_data) {
5203                 key_field = hist_data->fields[i];
5204                 key_field_test = hist_data_test->fields[i];
5205
5206                 if (key_field->flags != key_field_test->flags)
5207                         return false;
5208                 if (!compatible_field(key_field->field, key_field_test->field))
5209                         return false;
5210                 if (key_field->offset != key_field_test->offset)
5211                         return false;
5212                 if (key_field->size != key_field_test->size)
5213                         return false;
5214                 if (key_field->is_signed != key_field_test->is_signed)
5215                         return false;
5216                 if (!!key_field->var.name != !!key_field_test->var.name)
5217                         return false;
5218                 if (key_field->var.name &&
5219                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5220                         return false;
5221         }
5222
5223         for (i = 0; i < hist_data->n_sort_keys; i++) {
5224                 sort_key = &hist_data->sort_keys[i];
5225                 sort_key_test = &hist_data_test->sort_keys[i];
5226
5227                 if (sort_key->field_idx != sort_key_test->field_idx ||
5228                     sort_key->descending != sort_key_test->descending)
5229                         return false;
5230         }
5231
5232         if (!ignore_filter && data->filter_str &&
5233             (strcmp(data->filter_str, data_test->filter_str) != 0))
5234                 return false;
5235
5236         if (!actions_match(hist_data, hist_data_test))
5237                 return false;
5238
5239         return true;
5240 }
5241
5242 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5243                                  struct event_trigger_data *data,
5244                                  struct trace_event_file *file)
5245 {
5246         struct hist_trigger_data *hist_data = data->private_data;
5247         struct event_trigger_data *test, *named_data = NULL;
5248         int ret = 0;
5249
5250         if (hist_data->attrs->name) {
5251                 named_data = find_named_trigger(hist_data->attrs->name);
5252                 if (named_data) {
5253                         if (!hist_trigger_match(data, named_data, named_data,
5254                                                 true)) {
5255                                 hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name);
5256                                 ret = -EINVAL;
5257                                 goto out;
5258                         }
5259                 }
5260         }
5261
5262         if (hist_data->attrs->name && !named_data)
5263                 goto new;
5264
5265         list_for_each_entry_rcu(test, &file->triggers, list) {
5266                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5267                         if (!hist_trigger_match(data, test, named_data, false))
5268                                 continue;
5269                         if (hist_data->attrs->pause)
5270                                 test->paused = true;
5271                         else if (hist_data->attrs->cont)
5272                                 test->paused = false;
5273                         else if (hist_data->attrs->clear)
5274                                 hist_clear(test);
5275                         else {
5276                                 hist_err("Hist trigger already exists", NULL);
5277                                 ret = -EEXIST;
5278                         }
5279                         goto out;
5280                 }
5281         }
5282  new:
5283         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5284                 hist_err("Can't clear or continue a nonexistent hist trigger", NULL);
5285                 ret = -ENOENT;
5286                 goto out;
5287         }
5288
5289         if (hist_data->attrs->pause)
5290                 data->paused = true;
5291
5292         if (named_data) {
5293                 data->private_data = named_data->private_data;
5294                 set_named_trigger_data(data, named_data);
5295                 data->ops = &event_hist_trigger_named_ops;
5296         }
5297
5298         if (data->ops->init) {
5299                 ret = data->ops->init(data->ops, data);
5300                 if (ret < 0)
5301                         goto out;
5302         }
5303
5304         if (hist_data->enable_timestamps) {
5305                 char *clock = hist_data->attrs->clock;
5306
5307                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5308                 if (ret) {
5309                         hist_err("Couldn't set trace_clock: ", clock);
5310                         goto out;
5311                 }
5312
5313                 tracing_set_time_stamp_abs(file->tr, true);
5314         }
5315
5316         if (named_data)
5317                 destroy_hist_data(hist_data);
5318
5319         ret++;
5320  out:
5321         return ret;
5322 }
5323
5324 static int hist_trigger_enable(struct event_trigger_data *data,
5325                                struct trace_event_file *file)
5326 {
5327         int ret = 0;
5328
5329         list_add_tail_rcu(&data->list, &file->triggers);
5330
5331         update_cond_flag(file);
5332
5333         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5334                 list_del_rcu(&data->list);
5335                 update_cond_flag(file);
5336                 ret--;
5337         }
5338
5339         return ret;
5340 }
5341
5342 static bool have_hist_trigger_match(struct event_trigger_data *data,
5343                                     struct trace_event_file *file)
5344 {
5345         struct hist_trigger_data *hist_data = data->private_data;
5346         struct event_trigger_data *test, *named_data = NULL;
5347         bool match = false;
5348
5349         if (hist_data->attrs->name)
5350                 named_data = find_named_trigger(hist_data->attrs->name);
5351
5352         list_for_each_entry_rcu(test, &file->triggers, list) {
5353                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5354                         if (hist_trigger_match(data, test, named_data, false)) {
5355                                 match = true;
5356                                 break;
5357                         }
5358                 }
5359         }
5360
5361         return match;
5362 }
5363
5364 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5365                                     struct trace_event_file *file)
5366 {
5367         struct hist_trigger_data *hist_data = data->private_data;
5368         struct event_trigger_data *test, *named_data = NULL;
5369
5370         if (hist_data->attrs->name)
5371                 named_data = find_named_trigger(hist_data->attrs->name);
5372
5373         list_for_each_entry_rcu(test, &file->triggers, list) {
5374                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5375                         if (!hist_trigger_match(data, test, named_data, false))
5376                                 continue;
5377                         hist_data = test->private_data;
5378                         if (check_var_refs(hist_data))
5379                                 return true;
5380                         break;
5381                 }
5382         }
5383
5384         return false;
5385 }
5386
5387 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5388                                     struct event_trigger_data *data,
5389                                     struct trace_event_file *file)
5390 {
5391         struct hist_trigger_data *hist_data = data->private_data;
5392         struct event_trigger_data *test, *named_data = NULL;
5393         bool unregistered = false;
5394
5395         if (hist_data->attrs->name)
5396                 named_data = find_named_trigger(hist_data->attrs->name);
5397
5398         list_for_each_entry_rcu(test, &file->triggers, list) {
5399                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5400                         if (!hist_trigger_match(data, test, named_data, false))
5401                                 continue;
5402                         unregistered = true;
5403                         list_del_rcu(&test->list);
5404                         trace_event_trigger_enable_disable(file, 0);
5405                         update_cond_flag(file);
5406                         break;
5407                 }
5408         }
5409
5410         if (unregistered && test->ops->free)
5411                 test->ops->free(test->ops, test);
5412
5413         if (hist_data->enable_timestamps) {
5414                 if (!hist_data->remove || unregistered)
5415                         tracing_set_time_stamp_abs(file->tr, false);
5416         }
5417 }
5418
5419 static bool hist_file_check_refs(struct trace_event_file *file)
5420 {
5421         struct hist_trigger_data *hist_data;
5422         struct event_trigger_data *test;
5423
5424         list_for_each_entry_rcu(test, &file->triggers, list) {
5425                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5426                         hist_data = test->private_data;
5427                         if (check_var_refs(hist_data))
5428                                 return true;
5429                 }
5430         }
5431
5432         return false;
5433 }
5434
5435 static void hist_unreg_all(struct trace_event_file *file)
5436 {
5437         struct event_trigger_data *test, *n;
5438         struct hist_trigger_data *hist_data;
5439         struct synth_event *se;
5440         const char *se_name;
5441
5442         if (hist_file_check_refs(file))
5443                 return;
5444
5445         list_for_each_entry_safe(test, n, &file->triggers, list) {
5446                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5447                         hist_data = test->private_data;
5448                         list_del_rcu(&test->list);
5449                         trace_event_trigger_enable_disable(file, 0);
5450
5451                         mutex_lock(&synth_event_mutex);
5452                         se_name = trace_event_name(file->event_call);
5453                         se = find_synth_event(se_name);
5454                         if (se)
5455                                 se->ref--;
5456                         mutex_unlock(&synth_event_mutex);
5457
5458                         update_cond_flag(file);
5459                         if (hist_data->enable_timestamps)
5460                                 tracing_set_time_stamp_abs(file->tr, false);
5461                         if (test->ops->free)
5462                                 test->ops->free(test->ops, test);
5463                 }
5464         }
5465 }
5466
5467 static int event_hist_trigger_func(struct event_command *cmd_ops,
5468                                    struct trace_event_file *file,
5469                                    char *glob, char *cmd, char *param)
5470 {
5471         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5472         struct event_trigger_data *trigger_data;
5473         struct hist_trigger_attrs *attrs;
5474         struct event_trigger_ops *trigger_ops;
5475         struct hist_trigger_data *hist_data;
5476         struct synth_event *se;
5477         const char *se_name;
5478         bool remove = false;
5479         char *trigger, *p;
5480         int ret = 0;
5481
5482         if (glob && strlen(glob)) {
5483                 last_cmd_set(param);
5484                 hist_err_clear();
5485         }
5486
5487         if (!param)
5488                 return -EINVAL;
5489
5490         if (glob[0] == '!')
5491                 remove = true;
5492
5493         /*
5494          * separate the trigger from the filter (k:v [if filter])
5495          * allowing for whitespace in the trigger
5496          */
5497         p = trigger = param;
5498         do {
5499                 p = strstr(p, "if");
5500                 if (!p)
5501                         break;
5502                 if (p == param)
5503                         return -EINVAL;
5504                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5505                         p++;
5506                         continue;
5507                 }
5508                 if (p >= param + strlen(param) - strlen("if") - 1)
5509                         return -EINVAL;
5510                 if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') {
5511                         p++;
5512                         continue;
5513                 }
5514                 break;
5515         } while (p);
5516
5517         if (!p)
5518                 param = NULL;
5519         else {
5520                 *(p - 1) = '\0';
5521                 param = strstrip(p);
5522                 trigger = strstrip(trigger);
5523         }
5524
5525         attrs = parse_hist_trigger_attrs(trigger);
5526         if (IS_ERR(attrs))
5527                 return PTR_ERR(attrs);
5528
5529         if (attrs->map_bits)
5530                 hist_trigger_bits = attrs->map_bits;
5531
5532         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5533         if (IS_ERR(hist_data)) {
5534                 destroy_hist_trigger_attrs(attrs);
5535                 return PTR_ERR(hist_data);
5536         }
5537
5538         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5539
5540         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5541         if (!trigger_data) {
5542                 ret = -ENOMEM;
5543                 goto out_free;
5544         }
5545
5546         trigger_data->count = -1;
5547         trigger_data->ops = trigger_ops;
5548         trigger_data->cmd_ops = cmd_ops;
5549
5550         INIT_LIST_HEAD(&trigger_data->list);
5551         RCU_INIT_POINTER(trigger_data->filter, NULL);
5552
5553         trigger_data->private_data = hist_data;
5554
5555         /* if param is non-empty, it's supposed to be a filter */
5556         if (param && cmd_ops->set_filter) {
5557                 ret = cmd_ops->set_filter(param, trigger_data, file);
5558                 if (ret < 0)
5559                         goto out_free;
5560         }
5561
5562         if (remove) {
5563                 if (!have_hist_trigger_match(trigger_data, file))
5564                         goto out_free;
5565
5566                 if (hist_trigger_check_refs(trigger_data, file)) {
5567                         ret = -EBUSY;
5568                         goto out_free;
5569                 }
5570
5571                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5572
5573                 mutex_lock(&synth_event_mutex);
5574                 se_name = trace_event_name(file->event_call);
5575                 se = find_synth_event(se_name);
5576                 if (se)
5577                         se->ref--;
5578                 mutex_unlock(&synth_event_mutex);
5579
5580                 ret = 0;
5581                 goto out_free;
5582         }
5583
5584         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5585         /*
5586          * The above returns on success the # of triggers registered,
5587          * but if it didn't register any it returns zero.  Consider no
5588          * triggers registered a failure too.
5589          */
5590         if (!ret) {
5591                 if (!(attrs->pause || attrs->cont || attrs->clear))
5592                         ret = -ENOENT;
5593                 goto out_free;
5594         } else if (ret < 0)
5595                 goto out_free;
5596
5597         if (get_named_trigger_data(trigger_data))
5598                 goto enable;
5599
5600         if (has_hist_vars(hist_data))
5601                 save_hist_vars(hist_data);
5602
5603         ret = create_actions(hist_data, file);
5604         if (ret)
5605                 goto out_unreg;
5606
5607         ret = tracing_map_init(hist_data->map);
5608         if (ret)
5609                 goto out_unreg;
5610 enable:
5611         ret = hist_trigger_enable(trigger_data, file);
5612         if (ret)
5613                 goto out_unreg;
5614
5615         mutex_lock(&synth_event_mutex);
5616         se_name = trace_event_name(file->event_call);
5617         se = find_synth_event(se_name);
5618         if (se)
5619                 se->ref++;
5620         mutex_unlock(&synth_event_mutex);
5621
5622         /* Just return zero, not the number of registered triggers */
5623         ret = 0;
5624  out:
5625         if (ret == 0)
5626                 hist_err_clear();
5627
5628         return ret;
5629  out_unreg:
5630         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5631  out_free:
5632         if (cmd_ops->set_filter)
5633                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5634
5635         remove_hist_vars(hist_data);
5636
5637         kfree(trigger_data);
5638
5639         destroy_hist_data(hist_data);
5640         goto out;
5641 }
5642
5643 static struct event_command trigger_hist_cmd = {
5644         .name                   = "hist",
5645         .trigger_type           = ETT_EVENT_HIST,
5646         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5647         .func                   = event_hist_trigger_func,
5648         .reg                    = hist_register_trigger,
5649         .unreg                  = hist_unregister_trigger,
5650         .unreg_all              = hist_unreg_all,
5651         .get_trigger_ops        = event_hist_get_trigger_ops,
5652         .set_filter             = set_trigger_filter,
5653 };
5654
5655 __init int register_trigger_hist_cmd(void)
5656 {
5657         int ret;
5658
5659         ret = register_event_command(&trigger_hist_cmd);
5660         WARN_ON(ret < 0);
5661
5662         return ret;
5663 }
5664
5665 static void
5666 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5667                     struct ring_buffer_event *event)
5668 {
5669         struct enable_trigger_data *enable_data = data->private_data;
5670         struct event_trigger_data *test;
5671
5672         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
5673                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5674                         if (enable_data->enable)
5675                                 test->paused = false;
5676                         else
5677                                 test->paused = true;
5678                 }
5679         }
5680 }
5681
5682 static void
5683 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5684                           struct ring_buffer_event *event)
5685 {
5686         if (!data->count)
5687                 return;
5688
5689         if (data->count != -1)
5690                 (data->count)--;
5691
5692         hist_enable_trigger(data, rec, event);
5693 }
5694
5695 static struct event_trigger_ops hist_enable_trigger_ops = {
5696         .func                   = hist_enable_trigger,
5697         .print                  = event_enable_trigger_print,
5698         .init                   = event_trigger_init,
5699         .free                   = event_enable_trigger_free,
5700 };
5701
5702 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5703         .func                   = hist_enable_count_trigger,
5704         .print                  = event_enable_trigger_print,
5705         .init                   = event_trigger_init,
5706         .free                   = event_enable_trigger_free,
5707 };
5708
5709 static struct event_trigger_ops hist_disable_trigger_ops = {
5710         .func                   = hist_enable_trigger,
5711         .print                  = event_enable_trigger_print,
5712         .init                   = event_trigger_init,
5713         .free                   = event_enable_trigger_free,
5714 };
5715
5716 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5717         .func                   = hist_enable_count_trigger,
5718         .print                  = event_enable_trigger_print,
5719         .init                   = event_trigger_init,
5720         .free                   = event_enable_trigger_free,
5721 };
5722
5723 static struct event_trigger_ops *
5724 hist_enable_get_trigger_ops(char *cmd, char *param)
5725 {
5726         struct event_trigger_ops *ops;
5727         bool enable;
5728
5729         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5730
5731         if (enable)
5732                 ops = param ? &hist_enable_count_trigger_ops :
5733                         &hist_enable_trigger_ops;
5734         else
5735                 ops = param ? &hist_disable_count_trigger_ops :
5736                         &hist_disable_trigger_ops;
5737
5738         return ops;
5739 }
5740
5741 static void hist_enable_unreg_all(struct trace_event_file *file)
5742 {
5743         struct event_trigger_data *test, *n;
5744
5745         list_for_each_entry_safe(test, n, &file->triggers, list) {
5746                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5747                         list_del_rcu(&test->list);
5748                         update_cond_flag(file);
5749                         trace_event_trigger_enable_disable(file, 0);
5750                         if (test->ops->free)
5751                                 test->ops->free(test->ops, test);
5752                 }
5753         }
5754 }
5755
5756 static struct event_command trigger_hist_enable_cmd = {
5757         .name                   = ENABLE_HIST_STR,
5758         .trigger_type           = ETT_HIST_ENABLE,
5759         .func                   = event_enable_trigger_func,
5760         .reg                    = event_enable_register_trigger,
5761         .unreg                  = event_enable_unregister_trigger,
5762         .unreg_all              = hist_enable_unreg_all,
5763         .get_trigger_ops        = hist_enable_get_trigger_ops,
5764         .set_filter             = set_trigger_filter,
5765 };
5766
5767 static struct event_command trigger_hist_disable_cmd = {
5768         .name                   = DISABLE_HIST_STR,
5769         .trigger_type           = ETT_HIST_ENABLE,
5770         .func                   = event_enable_trigger_func,
5771         .reg                    = event_enable_register_trigger,
5772         .unreg                  = event_enable_unregister_trigger,
5773         .unreg_all              = hist_enable_unreg_all,
5774         .get_trigger_ops        = hist_enable_get_trigger_ops,
5775         .set_filter             = set_trigger_filter,
5776 };
5777
5778 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5779 {
5780         unregister_event_command(&trigger_hist_enable_cmd);
5781         unregister_event_command(&trigger_hist_disable_cmd);
5782 }
5783
5784 __init int register_trigger_hist_enable_disable_cmds(void)
5785 {
5786         int ret;
5787
5788         ret = register_event_command(&trigger_hist_enable_cmd);
5789         if (WARN_ON(ret < 0))
5790                 return ret;
5791         ret = register_event_command(&trigger_hist_disable_cmd);
5792         if (WARN_ON(ret < 0))
5793                 unregister_trigger_hist_enable_disable_cmds();
5794
5795         return ret;
5796 }
5797
5798 static __init int trace_events_hist_init(void)
5799 {
5800         struct dentry *entry = NULL;
5801         struct dentry *d_tracer;
5802         int err = 0;
5803
5804         d_tracer = tracing_init_dentry();
5805         if (IS_ERR(d_tracer)) {
5806                 err = PTR_ERR(d_tracer);
5807                 goto err;
5808         }
5809
5810         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
5811                                     NULL, &synth_events_fops);
5812         if (!entry) {
5813                 err = -ENODEV;
5814                 goto err;
5815         }
5816
5817         return err;
5818  err:
5819         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
5820
5821         return err;
5822 }
5823
5824 fs_initcall(trace_events_hist_init);