kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/mutex.h>
  11 #include <linux/slab.h>
  12 #include <linux/stacktrace.h>
  13 #include <linux/rculist.h>
  14 #include <linux/tracefs.h>
  15
  16 #include "tracing_map.h"
  17 #include "trace.h"
  18 #include "trace_dynevent.h"
  19
  20 #define SYNTH_SYSTEM            "synthetic"
  21 #define SYNTH_FIELDS_MAX        16
  22
  23 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  24
  25 struct hist_field;
  26
  27 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  28                                 struct tracing_map_elt *elt,
  29                                 struct ring_buffer_event *rbe,
  30                                 void *event);
  31
  32 #define HIST_FIELD_OPERANDS_MAX 2
  33 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  34 #define HIST_ACTIONS_MAX        8
  35
  36 enum field_op_id {
  37         FIELD_OP_NONE,
  38         FIELD_OP_PLUS,
  39         FIELD_OP_MINUS,
  40         FIELD_OP_UNARY_MINUS,
  41 };
  42
  43 /*
  44  * A hist_var (histogram variable) contains variable information for
  45  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
  46  * flag set.  A hist_var has a variable name e.g. ts0, and is
  47  * associated with a given histogram trigger, as specified by
  48  * hist_data.  The hist_var idx is the unique index assigned to the
  49  * variable by the hist trigger's tracing_map.  The idx is what is
  50  * used to set a variable's value and, by a variable reference, to
  51  * retrieve it.
  52  */
  53 struct hist_var {
  54         char                            *name;
  55         struct hist_trigger_data        *hist_data;
  56         unsigned int                    idx;
  57 };
  58
  59 struct hist_field {
  60         struct ftrace_event_field       *field;
  61         unsigned long                   flags;
  62         hist_field_fn_t                 fn;
  63         unsigned int                    size;
  64         unsigned int                    offset;
  65         unsigned int                    is_signed;
  66         const char                      *type;
  67         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
  68         struct hist_trigger_data        *hist_data;
  69
  70         /*
  71          * Variable fields contain variable-specific info in var.
  72          */
  73         struct hist_var                 var;
  74         enum field_op_id                operator;
  75         char                            *system;
  76         char                            *event_name;
  77
  78         /*
  79          * The name field is used for EXPR and VAR_REF fields.  VAR
  80          * fields contain the variable name in var.name.
  81          */
  82         char                            *name;
  83
  84         /*
  85          * When a histogram trigger is hit, if it has any references
  86          * to variables, the values of those variables are collected
  87          * into a var_ref_vals array by resolve_var_refs().  The
  88          * current value of each variable is read from the tracing_map
  89          * using the hist field's hist_var.idx and entered into the
  90          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
  91          */
  92         unsigned int                    var_ref_idx;
  93         bool                            read_once;
  94 };
  95
  96 static u64 hist_field_none(struct hist_field *field,
  97                            struct tracing_map_elt *elt,
  98                            struct ring_buffer_event *rbe,
  99                            void *event)
 100 {
 101         return 0;
 102 }
 103
 104 static u64 hist_field_counter(struct hist_field *field,
 105                               struct tracing_map_elt *elt,
 106                               struct ring_buffer_event *rbe,
 107                               void *event)
 108 {
 109         return 1;
 110 }
 111
 112 static u64 hist_field_string(struct hist_field *hist_field,
 113                              struct tracing_map_elt *elt,
 114                              struct ring_buffer_event *rbe,
 115                              void *event)
 116 {
 117         char *addr = (char *)(event + hist_field->field->offset);
 118
 119         return (u64)(unsigned long)addr;
 120 }
 121
 122 static u64 hist_field_dynstring(struct hist_field *hist_field,
 123                                 struct tracing_map_elt *elt,
 124                                 struct ring_buffer_event *rbe,
 125                                 void *event)
 126 {
 127         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 128         int str_loc = str_item & 0xffff;
 129         char *addr = (char *)(event + str_loc);
 130
 131         return (u64)(unsigned long)addr;
 132 }
 133
 134 static u64 hist_field_pstring(struct hist_field *hist_field,
 135                               struct tracing_map_elt *elt,
 136                               struct ring_buffer_event *rbe,
 137                               void *event)
 138 {
 139         char **addr = (char **)(event + hist_field->field->offset);
 140
 141         return (u64)(unsigned long)*addr;
 142 }
 143
 144 static u64 hist_field_log2(struct hist_field *hist_field,
 145                            struct tracing_map_elt *elt,
 146                            struct ring_buffer_event *rbe,
 147                            void *event)
 148 {
 149         struct hist_field *operand = hist_field->operands[0];
 150
 151         u64 val = operand->fn(operand, elt, rbe, event);
 152
 153         return (u64) ilog2(roundup_pow_of_two(val));
 154 }
 155
 156 static u64 hist_field_plus(struct hist_field *hist_field,
 157                            struct tracing_map_elt *elt,
 158                            struct ring_buffer_event *rbe,
 159                            void *event)
 160 {
 161         struct hist_field *operand1 = hist_field->operands[0];
 162         struct hist_field *operand2 = hist_field->operands[1];
 163
 164         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 165         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 166
 167         return val1 + val2;
 168 }
 169
 170 static u64 hist_field_minus(struct hist_field *hist_field,
 171                             struct tracing_map_elt *elt,
 172                             struct ring_buffer_event *rbe,
 173                             void *event)
 174 {
 175         struct hist_field *operand1 = hist_field->operands[0];
 176         struct hist_field *operand2 = hist_field->operands[1];
 177
 178         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 179         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 180
 181         return val1 - val2;
 182 }
 183
 184 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 185                                   struct tracing_map_elt *elt,
 186                                   struct ring_buffer_event *rbe,
 187                                   void *event)
 188 {
 189         struct hist_field *operand = hist_field->operands[0];
 190
 191         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 192         u64 val = (u64)-sval;
 193
 194         return val;
 195 }
 196
 197 #define DEFINE_HIST_FIELD_FN(type)                                      \
 198         static u64 hist_field_##type(struct hist_field *hist_field,     \
 199                                      struct tracing_map_elt *elt,       \
 200                                      struct ring_buffer_event *rbe,     \
 201                                      void *event)                       \
 202 {                                                                       \
 203         type *addr = (type *)(event + hist_field->field->offset);       \
 204                                                                         \
 205         return (u64)(unsigned long)*addr;                               \
 206 }
 207
 208 DEFINE_HIST_FIELD_FN(s64);
 209 DEFINE_HIST_FIELD_FN(u64);
 210 DEFINE_HIST_FIELD_FN(s32);
 211 DEFINE_HIST_FIELD_FN(u32);
 212 DEFINE_HIST_FIELD_FN(s16);
 213 DEFINE_HIST_FIELD_FN(u16);
 214 DEFINE_HIST_FIELD_FN(s8);
 215 DEFINE_HIST_FIELD_FN(u8);
 216
 217 #define for_each_hist_field(i, hist_data)       \
 218         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 219
 220 #define for_each_hist_val_field(i, hist_data)   \
 221         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 222
 223 #define for_each_hist_key_field(i, hist_data)   \
 224         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 225
 226 #define HIST_STACKTRACE_DEPTH   16
 227 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 228 #define HIST_STACKTRACE_SKIP    5
 229
 230 #define HITCOUNT_IDX            0
 231 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 232
 233 enum hist_field_flags {
 234         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 235         HIST_FIELD_FL_KEY               = 1 << 1,
 236         HIST_FIELD_FL_STRING            = 1 << 2,
 237         HIST_FIELD_FL_HEX               = 1 << 3,
 238         HIST_FIELD_FL_SYM               = 1 << 4,
 239         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 240         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 241         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 242         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 243         HIST_FIELD_FL_LOG2              = 1 << 9,
 244         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 245         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 246         HIST_FIELD_FL_VAR               = 1 << 12,
 247         HIST_FIELD_FL_EXPR              = 1 << 13,
 248         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 249         HIST_FIELD_FL_CPU               = 1 << 15,
 250         HIST_FIELD_FL_ALIAS             = 1 << 16,
 251 };
 252
 253 struct var_defs {
 254         unsigned int    n_vars;
 255         char            *name[TRACING_MAP_VARS_MAX];
 256         char            *expr[TRACING_MAP_VARS_MAX];
 257 };
 258
 259 struct hist_trigger_attrs {
 260         char            *keys_str;
 261         char            *vals_str;
 262         char            *sort_key_str;
 263         char            *name;
 264         char            *clock;
 265         bool            pause;
 266         bool            cont;
 267         bool            clear;
 268         bool            ts_in_usecs;
 269         unsigned int    map_bits;
 270
 271         char            *assignment_str[TRACING_MAP_VARS_MAX];
 272         unsigned int    n_assignments;
 273
 274         char            *action_str[HIST_ACTIONS_MAX];
 275         unsigned int    n_actions;
 276
 277         struct var_defs var_defs;
 278 };
 279
 280 struct field_var {
 281         struct hist_field       *var;
 282         struct hist_field       *val;
 283 };
 284
 285 struct field_var_hist {
 286         struct hist_trigger_data        *hist_data;
 287         char                            *cmd;
 288 };
 289
 290 struct hist_trigger_data {
 291         struct hist_field               *fields[HIST_FIELDS_MAX];
 292         unsigned int                    n_vals;
 293         unsigned int                    n_keys;
 294         unsigned int                    n_fields;
 295         unsigned int                    n_vars;
 296         unsigned int                    key_size;
 297         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 298         unsigned int                    n_sort_keys;
 299         struct trace_event_file         *event_file;
 300         struct hist_trigger_attrs       *attrs;
 301         struct tracing_map              *map;
 302         bool                            enable_timestamps;
 303         bool                            remove;
 304         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 305         unsigned int                    n_var_refs;
 306
 307         struct action_data              *actions[HIST_ACTIONS_MAX];
 308         unsigned int                    n_actions;
 309
 310         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 311         unsigned int                    n_field_vars;
 312         unsigned int                    n_field_var_str;
 313         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 314         unsigned int                    n_field_var_hists;
 315
 316         struct field_var                *max_vars[SYNTH_FIELDS_MAX];
 317         unsigned int                    n_max_vars;
 318         unsigned int                    n_max_var_str;
 319 };
 320
 321 static int synth_event_create(int argc, const char **argv);
 322 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 323 static int synth_event_release(struct dyn_event *ev);
 324 static bool synth_event_is_busy(struct dyn_event *ev);
 325 static bool synth_event_match(const char *system, const char *event,
 326                               struct dyn_event *ev);
 327
 328 static struct dyn_event_operations synth_event_ops = {
 329         .create = synth_event_create,
 330         .show = synth_event_show,
 331         .is_busy = synth_event_is_busy,
 332         .free = synth_event_release,
 333         .match = synth_event_match,
 334 };
 335
 336 struct synth_field {
 337         char *type;
 338         char *name;
 339         size_t size;
 340         bool is_signed;
 341         bool is_string;
 342 };
 343
 344 struct synth_event {
 345         struct dyn_event                        devent;
 346         int                                     ref;
 347         char                                    *name;
 348         struct synth_field                      **fields;
 349         unsigned int                            n_fields;
 350         unsigned int                            n_u64;
 351         struct trace_event_class                class;
 352         struct trace_event_call                 call;
 353         struct tracepoint                       *tp;
 354 };
 355
 356 static bool is_synth_event(struct dyn_event *ev)
 357 {
 358         return ev->ops == &synth_event_ops;
 359 }
 360
 361 static struct synth_event *to_synth_event(struct dyn_event *ev)
 362 {
 363         return container_of(ev, struct synth_event, devent);
 364 }
 365
 366 static bool synth_event_is_busy(struct dyn_event *ev)
 367 {
 368         struct synth_event *event = to_synth_event(ev);
 369
 370         return event->ref != 0;
 371 }
 372
 373 static bool synth_event_match(const char *system, const char *event,
 374                               struct dyn_event *ev)
 375 {
 376         struct synth_event *sev = to_synth_event(ev);
 377
 378         return strcmp(sev->name, event) == 0 &&
 379                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 380 }
 381
 382 struct action_data;
 383
 384 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 385                              struct tracing_map_elt *elt, void *rec,
 386                              struct ring_buffer_event *rbe,
 387                              struct action_data *data, u64 *var_ref_vals);
 388
 389 struct action_data {
 390         action_fn_t             fn;
 391         unsigned int            n_params;
 392         char                    *params[SYNTH_FIELDS_MAX];
 393
 394         union {
 395                 struct {
 396                         /*
 397                          * When a histogram trigger is hit, the values of any
 398                          * references to variables, including variables being passed
 399                          * as parameters to synthetic events, are collected into a
 400                          * var_ref_vals array.  This var_ref_idx is the index of the
 401                          * first param in the array to be passed to the synthetic
 402                          * event invocation.
 403                          */
 404                         unsigned int            var_ref_idx;
 405                         char                    *match_event;
 406                         char                    *match_event_system;
 407                         char                    *synth_event_name;
 408                         struct synth_event      *synth_event;
 409                 } onmatch;
 410
 411                 struct {
 412                         char                    *var_str;
 413                         char                    *fn_name;
 414                         unsigned int            max_var_ref_idx;
 415                         struct hist_field       *max_var;
 416                         struct hist_field       *var;
 417                 } onmax;
 418         };
 419 };
 420
 421
 422 static char last_hist_cmd[MAX_FILTER_STR_VAL];
 423 static char hist_err_str[MAX_FILTER_STR_VAL];
 424
 425 static void last_cmd_set(char *str)
 426 {
 427         if (!str)
 428                 return;
 429
 430         strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
 431 }
 432
 433 static void hist_err(char *str, char *var)
 434 {
 435         int maxlen = MAX_FILTER_STR_VAL - 1;
 436
 437         if (!str)
 438                 return;
 439
 440         if (strlen(hist_err_str))
 441                 return;
 442
 443         if (!var)
 444                 var = "";
 445
 446         if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
 447                 return;
 448
 449         strcat(hist_err_str, str);
 450         strcat(hist_err_str, var);
 451 }
 452
 453 static void hist_err_event(char *str, char *system, char *event, char *var)
 454 {
 455         char err[MAX_FILTER_STR_VAL];
 456
 457         if (system && var)
 458                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
 459         else if (system)
 460                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
 461         else
 462                 strscpy(err, var, MAX_FILTER_STR_VAL);
 463
 464         hist_err(str, err);
 465 }
 466
 467 static void hist_err_clear(void)
 468 {
 469         hist_err_str[0] = '\0';
 470 }
 471
 472 static bool have_hist_err(void)
 473 {
 474         if (strlen(hist_err_str))
 475                 return true;
 476
 477         return false;
 478 }
 479
 480 struct synth_trace_event {
 481         struct trace_entry      ent;
 482         u64                     fields[];
 483 };
 484
 485 static int synth_event_define_fields(struct trace_event_call *call)
 486 {
 487         struct synth_trace_event trace;
 488         int offset = offsetof(typeof(trace), fields);
 489         struct synth_event *event = call->data;
 490         unsigned int i, size, n_u64;
 491         char *name, *type;
 492         bool is_signed;
 493         int ret = 0;
 494
 495         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 496                 size = event->fields[i]->size;
 497                 is_signed = event->fields[i]->is_signed;
 498                 type = event->fields[i]->type;
 499                 name = event->fields[i]->name;
 500                 ret = trace_define_field(call, type, name, offset, size,
 501                                          is_signed, FILTER_OTHER);
 502                 if (ret)
 503                         break;
 504
 505                 if (event->fields[i]->is_string) {
 506                         offset += STR_VAR_LEN_MAX;
 507                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 508                 } else {
 509                         offset += sizeof(u64);
 510                         n_u64++;
 511                 }
 512         }
 513
 514         event->n_u64 = n_u64;
 515
 516         return ret;
 517 }
 518
 519 static bool synth_field_signed(char *type)
 520 {
 521         if (str_has_prefix(type, "u"))
 522                 return false;
 523
 524         return true;
 525 }
 526
 527 static int synth_field_is_string(char *type)
 528 {
 529         if (strstr(type, "char[") != NULL)
 530                 return true;
 531
 532         return false;
 533 }
 534
 535 static int synth_field_string_size(char *type)
 536 {
 537         char buf[4], *end, *start;
 538         unsigned int len;
 539         int size, err;
 540
 541         start = strstr(type, "char[");
 542         if (start == NULL)
 543                 return -EINVAL;
 544         start += sizeof("char[") - 1;
 545
 546         end = strchr(type, ']');
 547         if (!end || end < start)
 548                 return -EINVAL;
 549
 550         len = end - start;
 551         if (len > 3)
 552                 return -EINVAL;
 553
 554         strncpy(buf, start, len);
 555         buf[len] = '\0';
 556
 557         err = kstrtouint(buf, 0, &size);
 558         if (err)
 559                 return err;
 560
 561         if (size > STR_VAR_LEN_MAX)
 562                 return -EINVAL;
 563
 564         return size;
 565 }
 566
 567 static int synth_field_size(char *type)
 568 {
 569         int size = 0;
 570
 571         if (strcmp(type, "s64") == 0)
 572                 size = sizeof(s64);
 573         else if (strcmp(type, "u64") == 0)
 574                 size = sizeof(u64);
 575         else if (strcmp(type, "s32") == 0)
 576                 size = sizeof(s32);
 577         else if (strcmp(type, "u32") == 0)
 578                 size = sizeof(u32);
 579         else if (strcmp(type, "s16") == 0)
 580                 size = sizeof(s16);
 581         else if (strcmp(type, "u16") == 0)
 582                 size = sizeof(u16);
 583         else if (strcmp(type, "s8") == 0)
 584                 size = sizeof(s8);
 585         else if (strcmp(type, "u8") == 0)
 586                 size = sizeof(u8);
 587         else if (strcmp(type, "char") == 0)
 588                 size = sizeof(char);
 589         else if (strcmp(type, "unsigned char") == 0)
 590                 size = sizeof(unsigned char);
 591         else if (strcmp(type, "int") == 0)
 592                 size = sizeof(int);
 593         else if (strcmp(type, "unsigned int") == 0)
 594                 size = sizeof(unsigned int);
 595         else if (strcmp(type, "long") == 0)
 596                 size = sizeof(long);
 597         else if (strcmp(type, "unsigned long") == 0)
 598                 size = sizeof(unsigned long);
 599         else if (strcmp(type, "pid_t") == 0)
 600                 size = sizeof(pid_t);
 601         else if (synth_field_is_string(type))
 602                 size = synth_field_string_size(type);
 603
 604         return size;
 605 }
 606
 607 static const char *synth_field_fmt(char *type)
 608 {
 609         const char *fmt = "%llu";
 610
 611         if (strcmp(type, "s64") == 0)
 612                 fmt = "%lld";
 613         else if (strcmp(type, "u64") == 0)
 614                 fmt = "%llu";
 615         else if (strcmp(type, "s32") == 0)
 616                 fmt = "%d";
 617         else if (strcmp(type, "u32") == 0)
 618                 fmt = "%u";
 619         else if (strcmp(type, "s16") == 0)
 620                 fmt = "%d";
 621         else if (strcmp(type, "u16") == 0)
 622                 fmt = "%u";
 623         else if (strcmp(type, "s8") == 0)
 624                 fmt = "%d";
 625         else if (strcmp(type, "u8") == 0)
 626                 fmt = "%u";
 627         else if (strcmp(type, "char") == 0)
 628                 fmt = "%d";
 629         else if (strcmp(type, "unsigned char") == 0)
 630                 fmt = "%u";
 631         else if (strcmp(type, "int") == 0)
 632                 fmt = "%d";
 633         else if (strcmp(type, "unsigned int") == 0)
 634                 fmt = "%u";
 635         else if (strcmp(type, "long") == 0)
 636                 fmt = "%ld";
 637         else if (strcmp(type, "unsigned long") == 0)
 638                 fmt = "%lu";
 639         else if (strcmp(type, "pid_t") == 0)
 640                 fmt = "%d";
 641         else if (synth_field_is_string(type))
 642                 fmt = "%s";
 643
 644         return fmt;
 645 }
 646
 647 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 648                                            int flags,
 649                                            struct trace_event *event)
 650 {
 651         struct trace_array *tr = iter->tr;
 652         struct trace_seq *s = &iter->seq;
 653         struct synth_trace_event *entry;
 654         struct synth_event *se;
 655         unsigned int i, n_u64;
 656         char print_fmt[32];
 657         const char *fmt;
 658
 659         entry = (struct synth_trace_event *)iter->ent;
 660         se = container_of(event, struct synth_event, call.event);
 661
 662         trace_seq_printf(s, "%s: ", se->name);
 663
 664         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 665                 if (trace_seq_has_overflowed(s))
 666                         goto end;
 667
 668                 fmt = synth_field_fmt(se->fields[i]->type);
 669
 670                 /* parameter types */
 671                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 672                         trace_seq_printf(s, "%s ", fmt);
 673
 674                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 675
 676                 /* parameter values */
 677                 if (se->fields[i]->is_string) {
 678                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 679                                          (char *)&entry->fields[n_u64],
 680                                          i == se->n_fields - 1 ? "" : " ");
 681                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 682                 } else {
 683                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 684                                          entry->fields[n_u64],
 685                                          i == se->n_fields - 1 ? "" : " ");
 686                         n_u64++;
 687                 }
 688         }
 689 end:
 690         trace_seq_putc(s, '\n');
 691
 692         return trace_handle_return(s);
 693 }
 694
 695 static struct trace_event_functions synth_event_funcs = {
 696         .trace          = print_synth_event
 697 };
 698
 699 static notrace void trace_event_raw_event_synth(void *__data,
 700                                                 u64 *var_ref_vals,
 701                                                 unsigned int var_ref_idx)
 702 {
 703         struct trace_event_file *trace_file = __data;
 704         struct synth_trace_event *entry;
 705         struct trace_event_buffer fbuffer;
 706         struct ring_buffer *buffer;
 707         struct synth_event *event;
 708         unsigned int i, n_u64;
 709         int fields_size = 0;
 710
 711         event = trace_file->event_call->data;
 712
 713         if (trace_trigger_soft_disabled(trace_file))
 714                 return;
 715
 716         fields_size = event->n_u64 * sizeof(u64);
 717
 718         /*
 719          * Avoid ring buffer recursion detection, as this event
 720          * is being performed within another event.
 721          */
 722         buffer = trace_file->tr->trace_buffer.buffer;
 723         ring_buffer_nest_start(buffer);
 724
 725         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 726                                            sizeof(*entry) + fields_size);
 727         if (!entry)
 728                 goto out;
 729
 730         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 731                 if (event->fields[i]->is_string) {
 732                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 733                         char *str_field = (char *)&entry->fields[n_u64];
 734
 735                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 736                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 737                 } else {
 738                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 739                         n_u64++;
 740                 }
 741         }
 742
 743         trace_event_buffer_commit(&fbuffer);
 744 out:
 745         ring_buffer_nest_end(buffer);
 746 }
 747
 748 static void free_synth_event_print_fmt(struct trace_event_call *call)
 749 {
 750         if (call) {
 751                 kfree(call->print_fmt);
 752                 call->print_fmt = NULL;
 753         }
 754 }
 755
 756 static int __set_synth_event_print_fmt(struct synth_event *event,
 757                                        char *buf, int len)
 758 {
 759         const char *fmt;
 760         int pos = 0;
 761         int i;
 762
 763         /* When len=0, we just calculate the needed length */
 764 #define LEN_OR_ZERO (len ? len - pos : 0)
 765
 766         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 767         for (i = 0; i < event->n_fields; i++) {
 768                 fmt = synth_field_fmt(event->fields[i]->type);
 769                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 770                                 event->fields[i]->name, fmt,
 771                                 i == event->n_fields - 1 ? "" : ", ");
 772         }
 773         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 774
 775         for (i = 0; i < event->n_fields; i++) {
 776                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 777                                 ", REC->%s", event->fields[i]->name);
 778         }
 779
 780 #undef LEN_OR_ZERO
 781
 782         /* return the length of print_fmt */
 783         return pos;
 784 }
 785
 786 static int set_synth_event_print_fmt(struct trace_event_call *call)
 787 {
 788         struct synth_event *event = call->data;
 789         char *print_fmt;
 790         int len;
 791
 792         /* First: called with 0 length to calculate the needed length */
 793         len = __set_synth_event_print_fmt(event, NULL, 0);
 794
 795         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 796         if (!print_fmt)
 797                 return -ENOMEM;
 798
 799         /* Second: actually write the @print_fmt */
 800         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 801         call->print_fmt = print_fmt;
 802
 803         return 0;
 804 }
 805
 806 static void free_synth_field(struct synth_field *field)
 807 {
 808         kfree(field->type);
 809         kfree(field->name);
 810         kfree(field);
 811 }
 812
 813 static struct synth_field *parse_synth_field(int argc, const char **argv,
 814                                              int *consumed)
 815 {
 816         struct synth_field *field;
 817         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
 818         int len, ret = 0;
 819
 820         if (field_type[0] == ';')
 821                 field_type++;
 822
 823         if (!strcmp(field_type, "unsigned")) {
 824                 if (argc < 3)
 825                         return ERR_PTR(-EINVAL);
 826                 prefix = "unsigned ";
 827                 field_type = argv[1];
 828                 field_name = argv[2];
 829                 *consumed = 3;
 830         } else {
 831                 field_name = argv[1];
 832                 *consumed = 2;
 833         }
 834
 835         field = kzalloc(sizeof(*field), GFP_KERNEL);
 836         if (!field)
 837                 return ERR_PTR(-ENOMEM);
 838
 839         len = strlen(field_name);
 840         array = strchr(field_name, '[');
 841         if (array)
 842                 len -= strlen(array);
 843         else if (field_name[len - 1] == ';')
 844                 len--;
 845
 846         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
 847         if (!field->name) {
 848                 ret = -ENOMEM;
 849                 goto free;
 850         }
 851
 852         if (field_type[0] == ';')
 853                 field_type++;
 854         len = strlen(field_type) + 1;
 855         if (array)
 856                 len += strlen(array);
 857         if (prefix)
 858                 len += strlen(prefix);
 859
 860         field->type = kzalloc(len, GFP_KERNEL);
 861         if (!field->type) {
 862                 ret = -ENOMEM;
 863                 goto free;
 864         }
 865         if (prefix)
 866                 strcat(field->type, prefix);
 867         strcat(field->type, field_type);
 868         if (array) {
 869                 strcat(field->type, array);
 870                 if (field->type[len - 1] == ';')
 871                         field->type[len - 1] = '\0';
 872         }
 873
 874         field->size = synth_field_size(field->type);
 875         if (!field->size) {
 876                 ret = -EINVAL;
 877                 goto free;
 878         }
 879
 880         if (synth_field_is_string(field->type))
 881                 field->is_string = true;
 882
 883         field->is_signed = synth_field_signed(field->type);
 884
 885  out:
 886         return field;
 887  free:
 888         free_synth_field(field);
 889         field = ERR_PTR(ret);
 890         goto out;
 891 }
 892
 893 static void free_synth_tracepoint(struct tracepoint *tp)
 894 {
 895         if (!tp)
 896                 return;
 897
 898         kfree(tp->name);
 899         kfree(tp);
 900 }
 901
 902 static struct tracepoint *alloc_synth_tracepoint(char *name)
 903 {
 904         struct tracepoint *tp;
 905
 906         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 907         if (!tp)
 908                 return ERR_PTR(-ENOMEM);
 909
 910         tp->name = kstrdup(name, GFP_KERNEL);
 911         if (!tp->name) {
 912                 kfree(tp);
 913                 return ERR_PTR(-ENOMEM);
 914         }
 915
 916         return tp;
 917 }
 918
 919 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 920                                     unsigned int var_ref_idx);
 921
 922 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 923                                unsigned int var_ref_idx)
 924 {
 925         struct tracepoint *tp = event->tp;
 926
 927         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 928                 struct tracepoint_func *probe_func_ptr;
 929                 synth_probe_func_t probe_func;
 930                 void *__data;
 931
 932                 if (!(cpu_online(raw_smp_processor_id())))
 933                         return;
 934
 935                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 936                 if (probe_func_ptr) {
 937                         do {
 938                                 probe_func = probe_func_ptr->func;
 939                                 __data = probe_func_ptr->data;
 940                                 probe_func(__data, var_ref_vals, var_ref_idx);
 941                         } while ((++probe_func_ptr)->func);
 942                 }
 943         }
 944 }
 945
 946 static struct synth_event *find_synth_event(const char *name)
 947 {
 948         struct dyn_event *pos;
 949         struct synth_event *event;
 950
 951         for_each_dyn_event(pos) {
 952                 if (!is_synth_event(pos))
 953                         continue;
 954                 event = to_synth_event(pos);
 955                 if (strcmp(event->name, name) == 0)
 956                         return event;
 957         }
 958
 959         return NULL;
 960 }
 961
 962 static int register_synth_event(struct synth_event *event)
 963 {
 964         struct trace_event_call *call = &event->call;
 965         int ret = 0;
 966
 967         event->call.class = &event->class;
 968         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
 969         if (!event->class.system) {
 970                 ret = -ENOMEM;
 971                 goto out;
 972         }
 973
 974         event->tp = alloc_synth_tracepoint(event->name);
 975         if (IS_ERR(event->tp)) {
 976                 ret = PTR_ERR(event->tp);
 977                 event->tp = NULL;
 978                 goto out;
 979         }
 980
 981         INIT_LIST_HEAD(&call->class->fields);
 982         call->event.funcs = &synth_event_funcs;
 983         call->class->define_fields = synth_event_define_fields;
 984
 985         ret = register_trace_event(&call->event);
 986         if (!ret) {
 987                 ret = -ENODEV;
 988                 goto out;
 989         }
 990         call->flags = TRACE_EVENT_FL_TRACEPOINT;
 991         call->class->reg = trace_event_reg;
 992         call->class->probe = trace_event_raw_event_synth;
 993         call->data = event;
 994         call->tp = event->tp;
 995
 996         ret = trace_add_event_call(call);
 997         if (ret) {
 998                 pr_warn("Failed to register synthetic event: %s\n",
 999                         trace_event_name(call));
1000                 goto err;
1001         }
1002
1003         ret = set_synth_event_print_fmt(call);
1004         if (ret < 0) {
1005                 trace_remove_event_call(call);
1006                 goto err;
1007         }
1008  out:
1009         return ret;
1010  err:
1011         unregister_trace_event(&call->event);
1012         goto out;
1013 }
1014
1015 static int unregister_synth_event(struct synth_event *event)
1016 {
1017         struct trace_event_call *call = &event->call;
1018         int ret;
1019
1020         ret = trace_remove_event_call(call);
1021
1022         return ret;
1023 }
1024
1025 static void free_synth_event(struct synth_event *event)
1026 {
1027         unsigned int i;
1028
1029         if (!event)
1030                 return;
1031
1032         for (i = 0; i < event->n_fields; i++)
1033                 free_synth_field(event->fields[i]);
1034
1035         kfree(event->fields);
1036         kfree(event->name);
1037         kfree(event->class.system);
1038         free_synth_tracepoint(event->tp);
1039         free_synth_event_print_fmt(&event->call);
1040         kfree(event);
1041 }
1042
1043 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1044                                              struct synth_field **fields)
1045 {
1046         struct synth_event *event;
1047         unsigned int i;
1048
1049         event = kzalloc(sizeof(*event), GFP_KERNEL);
1050         if (!event) {
1051                 event = ERR_PTR(-ENOMEM);
1052                 goto out;
1053         }
1054
1055         event->name = kstrdup(name, GFP_KERNEL);
1056         if (!event->name) {
1057                 kfree(event);
1058                 event = ERR_PTR(-ENOMEM);
1059                 goto out;
1060         }
1061
1062         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1063         if (!event->fields) {
1064                 free_synth_event(event);
1065                 event = ERR_PTR(-ENOMEM);
1066                 goto out;
1067         }
1068
1069         dyn_event_init(&event->devent, &synth_event_ops);
1070
1071         for (i = 0; i < n_fields; i++)
1072                 event->fields[i] = fields[i];
1073
1074         event->n_fields = n_fields;
1075  out:
1076         return event;
1077 }
1078
1079 static void action_trace(struct hist_trigger_data *hist_data,
1080                          struct tracing_map_elt *elt, void *rec,
1081                          struct ring_buffer_event *rbe,
1082                          struct action_data *data, u64 *var_ref_vals)
1083 {
1084         struct synth_event *event = data->onmatch.synth_event;
1085
1086         trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx);
1087 }
1088
1089 struct hist_var_data {
1090         struct list_head list;
1091         struct hist_trigger_data *hist_data;
1092 };
1093
1094 static int __create_synth_event(int argc, const char *name, const char **argv)
1095 {
1096         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1097         struct synth_event *event = NULL;
1098         int i, consumed = 0, n_fields = 0, ret = 0;
1099
1100         /*
1101          * Argument syntax:
1102          *  - Add synthetic event: <event_name> field[;field] ...
1103          *  - Remove synthetic event: !<event_name> field[;field] ...
1104          *      where 'field' = type field_name
1105          */
1106
1107         if (name[0] == '\0' || argc < 1)
1108                 return -EINVAL;
1109
1110         mutex_lock(&event_mutex);
1111
1112         event = find_synth_event(name);
1113         if (event) {
1114                 ret = -EEXIST;
1115                 goto out;
1116         }
1117
1118         for (i = 0; i < argc - 1; i++) {
1119                 if (strcmp(argv[i], ";") == 0)
1120                         continue;
1121                 if (n_fields == SYNTH_FIELDS_MAX) {
1122                         ret = -EINVAL;
1123                         goto err;
1124                 }
1125
1126                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1127                 if (IS_ERR(field)) {
1128                         ret = PTR_ERR(field);
1129                         goto err;
1130                 }
1131                 fields[n_fields++] = field;
1132                 i += consumed - 1;
1133         }
1134
1135         if (i < argc && strcmp(argv[i], ";") != 0) {
1136                 ret = -EINVAL;
1137                 goto err;
1138         }
1139
1140         event = alloc_synth_event(name, n_fields, fields);
1141         if (IS_ERR(event)) {
1142                 ret = PTR_ERR(event);
1143                 event = NULL;
1144                 goto err;
1145         }
1146         ret = register_synth_event(event);
1147         if (!ret)
1148                 dyn_event_add(&event->devent);
1149         else
1150                 free_synth_event(event);
1151  out:
1152         mutex_unlock(&event_mutex);
1153
1154         return ret;
1155  err:
1156         for (i = 0; i < n_fields; i++)
1157                 free_synth_field(fields[i]);
1158
1159         goto out;
1160 }
1161
1162 static int create_or_delete_synth_event(int argc, char **argv)
1163 {
1164         const char *name = argv[0];
1165         struct synth_event *event = NULL;
1166         int ret;
1167
1168         /* trace_run_command() ensures argc != 0 */
1169         if (name[0] == '!') {
1170                 mutex_lock(&event_mutex);
1171                 event = find_synth_event(name + 1);
1172                 if (event) {
1173                         if (event->ref)
1174                                 ret = -EBUSY;
1175                         else {
1176                                 ret = unregister_synth_event(event);
1177                                 if (!ret) {
1178                                         dyn_event_remove(&event->devent);
1179                                         free_synth_event(event);
1180                                 }
1181                         }
1182                 } else
1183                         ret = -ENOENT;
1184                 mutex_unlock(&event_mutex);
1185                 return ret;
1186         }
1187
1188         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1189         return ret == -ECANCELED ? -EINVAL : ret;
1190 }
1191
1192 static int synth_event_create(int argc, const char **argv)
1193 {
1194         const char *name = argv[0];
1195         int len;
1196
1197         if (name[0] != 's' || name[1] != ':')
1198                 return -ECANCELED;
1199         name += 2;
1200
1201         /* This interface accepts group name prefix */
1202         if (strchr(name, '/')) {
1203                 len = sizeof(SYNTH_SYSTEM "/") - 1;
1204                 if (strncmp(name, SYNTH_SYSTEM "/", len))
1205                         return -EINVAL;
1206                 name += len;
1207         }
1208         return __create_synth_event(argc - 1, name, argv + 1);
1209 }
1210
1211 static int synth_event_release(struct dyn_event *ev)
1212 {
1213         struct synth_event *event = to_synth_event(ev);
1214         int ret;
1215
1216         if (event->ref)
1217                 return -EBUSY;
1218
1219         ret = unregister_synth_event(event);
1220         if (ret)
1221                 return ret;
1222
1223         dyn_event_remove(ev);
1224         free_synth_event(event);
1225         return 0;
1226 }
1227
1228 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1229 {
1230         struct synth_field *field;
1231         unsigned int i;
1232
1233         seq_printf(m, "%s\t", event->name);
1234
1235         for (i = 0; i < event->n_fields; i++) {
1236                 field = event->fields[i];
1237
1238                 /* parameter values */
1239                 seq_printf(m, "%s %s%s", field->type, field->name,
1240                            i == event->n_fields - 1 ? "" : "; ");
1241         }
1242
1243         seq_putc(m, '\n');
1244
1245         return 0;
1246 }
1247
1248 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1249 {
1250         struct synth_event *event = to_synth_event(ev);
1251
1252         seq_printf(m, "s:%s/", event->class.system);
1253
1254         return __synth_event_show(m, event);
1255 }
1256
1257 static int synth_events_seq_show(struct seq_file *m, void *v)
1258 {
1259         struct dyn_event *ev = v;
1260
1261         if (!is_synth_event(ev))
1262                 return 0;
1263
1264         return __synth_event_show(m, to_synth_event(ev));
1265 }
1266
1267 static const struct seq_operations synth_events_seq_op = {
1268         .start  = dyn_event_seq_start,
1269         .next   = dyn_event_seq_next,
1270         .stop   = dyn_event_seq_stop,
1271         .show   = synth_events_seq_show,
1272 };
1273
1274 static int synth_events_open(struct inode *inode, struct file *file)
1275 {
1276         int ret;
1277
1278         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1279                 ret = dyn_events_release_all(&synth_event_ops);
1280                 if (ret < 0)
1281                         return ret;
1282         }
1283
1284         return seq_open(file, &synth_events_seq_op);
1285 }
1286
1287 static ssize_t synth_events_write(struct file *file,
1288                                   const char __user *buffer,
1289                                   size_t count, loff_t *ppos)
1290 {
1291         return trace_parse_run_command(file, buffer, count, ppos,
1292                                        create_or_delete_synth_event);
1293 }
1294
1295 static const struct file_operations synth_events_fops = {
1296         .open           = synth_events_open,
1297         .write          = synth_events_write,
1298         .read           = seq_read,
1299         .llseek         = seq_lseek,
1300         .release        = seq_release,
1301 };
1302
1303 static u64 hist_field_timestamp(struct hist_field *hist_field,
1304                                 struct tracing_map_elt *elt,
1305                                 struct ring_buffer_event *rbe,
1306                                 void *event)
1307 {
1308         struct hist_trigger_data *hist_data = hist_field->hist_data;
1309         struct trace_array *tr = hist_data->event_file->tr;
1310
1311         u64 ts = ring_buffer_event_time_stamp(rbe);
1312
1313         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1314                 ts = ns2usecs(ts);
1315
1316         return ts;
1317 }
1318
1319 static u64 hist_field_cpu(struct hist_field *hist_field,
1320                           struct tracing_map_elt *elt,
1321                           struct ring_buffer_event *rbe,
1322                           void *event)
1323 {
1324         int cpu = smp_processor_id();
1325
1326         return cpu;
1327 }
1328
1329 /**
1330  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1331  * @hist_field: The VAR_REF field to check
1332  * @var_data: The hist trigger that owns the variable
1333  * @var_idx: The trigger variable identifier
1334  *
1335  * Check the given VAR_REF field to see whether or not it references
1336  * the given variable associated with the given trigger.
1337  *
1338  * Return: The VAR_REF field if it does reference the variable, NULL if not
1339  */
1340 static struct hist_field *
1341 check_field_for_var_ref(struct hist_field *hist_field,
1342                         struct hist_trigger_data *var_data,
1343                         unsigned int var_idx)
1344 {
1345         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1346
1347         if (hist_field && hist_field->var.idx == var_idx &&
1348             hist_field->var.hist_data == var_data)
1349                 return hist_field;
1350
1351         return NULL;
1352 }
1353
1354 /**
1355  * find_var_ref - Check if a trigger has a reference to a trigger variable
1356  * @hist_data: The hist trigger that might have a reference to the variable
1357  * @var_data: The hist trigger that owns the variable
1358  * @var_idx: The trigger variable identifier
1359  *
1360  * Check the list of var_refs[] on the first hist trigger to see
1361  * whether any of them are references to the variable on the second
1362  * trigger.
1363  *
1364  * Return: The VAR_REF field referencing the variable if so, NULL if not
1365  */
1366 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1367                                        struct hist_trigger_data *var_data,
1368                                        unsigned int var_idx)
1369 {
1370         struct hist_field *hist_field;
1371         unsigned int i;
1372
1373         for (i = 0; i < hist_data->n_var_refs; i++) {
1374                 hist_field = hist_data->var_refs[i];
1375                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1376                         return hist_field;
1377         }
1378
1379         return NULL;
1380 }
1381
1382 /**
1383  * find_any_var_ref - Check if there is a reference to a given trigger variable
1384  * @hist_data: The hist trigger
1385  * @var_idx: The trigger variable identifier
1386  *
1387  * Check to see whether the given variable is currently referenced by
1388  * any other trigger.
1389  *
1390  * The trigger the variable is defined on is explicitly excluded - the
1391  * assumption being that a self-reference doesn't prevent a trigger
1392  * from being removed.
1393  *
1394  * Return: The VAR_REF field referencing the variable if so, NULL if not
1395  */
1396 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1397                                            unsigned int var_idx)
1398 {
1399         struct trace_array *tr = hist_data->event_file->tr;
1400         struct hist_field *found = NULL;
1401         struct hist_var_data *var_data;
1402
1403         list_for_each_entry(var_data, &tr->hist_vars, list) {
1404                 if (var_data->hist_data == hist_data)
1405                         continue;
1406                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1407                 if (found)
1408                         break;
1409         }
1410
1411         return found;
1412 }
1413
1414 /**
1415  * check_var_refs - Check if there is a reference to any of trigger's variables
1416  * @hist_data: The hist trigger
1417  *
1418  * A trigger can define one or more variables.  If any one of them is
1419  * currently referenced by any other trigger, this function will
1420  * determine that.
1421
1422  * Typically used to determine whether or not a trigger can be removed
1423  * - if there are any references to a trigger's variables, it cannot.
1424  *
1425  * Return: True if there is a reference to any of trigger's variables
1426  */
1427 static bool check_var_refs(struct hist_trigger_data *hist_data)
1428 {
1429         struct hist_field *field;
1430         bool found = false;
1431         int i;
1432
1433         for_each_hist_field(i, hist_data) {
1434                 field = hist_data->fields[i];
1435                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1436                         if (find_any_var_ref(hist_data, field->var.idx)) {
1437                                 found = true;
1438                                 break;
1439                         }
1440                 }
1441         }
1442
1443         return found;
1444 }
1445
1446 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1447 {
1448         struct trace_array *tr = hist_data->event_file->tr;
1449         struct hist_var_data *var_data, *found = NULL;
1450
1451         list_for_each_entry(var_data, &tr->hist_vars, list) {
1452                 if (var_data->hist_data == hist_data) {
1453                         found = var_data;
1454                         break;
1455                 }
1456         }
1457
1458         return found;
1459 }
1460
1461 static bool field_has_hist_vars(struct hist_field *hist_field,
1462                                 unsigned int level)
1463 {
1464         int i;
1465
1466         if (level > 3)
1467                 return false;
1468
1469         if (!hist_field)
1470                 return false;
1471
1472         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1473             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1474                 return true;
1475
1476         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1477                 struct hist_field *operand;
1478
1479                 operand = hist_field->operands[i];
1480                 if (field_has_hist_vars(operand, level + 1))
1481                         return true;
1482         }
1483
1484         return false;
1485 }
1486
1487 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1488 {
1489         struct hist_field *hist_field;
1490         int i;
1491
1492         for_each_hist_field(i, hist_data) {
1493                 hist_field = hist_data->fields[i];
1494                 if (field_has_hist_vars(hist_field, 0))
1495                         return true;
1496         }
1497
1498         return false;
1499 }
1500
1501 static int save_hist_vars(struct hist_trigger_data *hist_data)
1502 {
1503         struct trace_array *tr = hist_data->event_file->tr;
1504         struct hist_var_data *var_data;
1505
1506         var_data = find_hist_vars(hist_data);
1507         if (var_data)
1508                 return 0;
1509
1510         if (trace_array_get(tr) < 0)
1511                 return -ENODEV;
1512
1513         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1514         if (!var_data) {
1515                 trace_array_put(tr);
1516                 return -ENOMEM;
1517         }
1518
1519         var_data->hist_data = hist_data;
1520         list_add(&var_data->list, &tr->hist_vars);
1521
1522         return 0;
1523 }
1524
1525 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1526 {
1527         struct trace_array *tr = hist_data->event_file->tr;
1528         struct hist_var_data *var_data;
1529
1530         var_data = find_hist_vars(hist_data);
1531         if (!var_data)
1532                 return;
1533
1534         if (WARN_ON(check_var_refs(hist_data)))
1535                 return;
1536
1537         list_del(&var_data->list);
1538
1539         kfree(var_data);
1540
1541         trace_array_put(tr);
1542 }
1543
1544 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1545                                          const char *var_name)
1546 {
1547         struct hist_field *hist_field, *found = NULL;
1548         int i;
1549
1550         for_each_hist_field(i, hist_data) {
1551                 hist_field = hist_data->fields[i];
1552                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1553                     strcmp(hist_field->var.name, var_name) == 0) {
1554                         found = hist_field;
1555                         break;
1556                 }
1557         }
1558
1559         return found;
1560 }
1561
1562 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1563                                    struct trace_event_file *file,
1564                                    const char *var_name)
1565 {
1566         struct hist_trigger_data *test_data;
1567         struct event_trigger_data *test;
1568         struct hist_field *hist_field;
1569
1570         hist_field = find_var_field(hist_data, var_name);
1571         if (hist_field)
1572                 return hist_field;
1573
1574         list_for_each_entry_rcu(test, &file->triggers, list) {
1575                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1576                         test_data = test->private_data;
1577                         hist_field = find_var_field(test_data, var_name);
1578                         if (hist_field)
1579                                 return hist_field;
1580                 }
1581         }
1582
1583         return NULL;
1584 }
1585
1586 static struct trace_event_file *find_var_file(struct trace_array *tr,
1587                                               char *system,
1588                                               char *event_name,
1589                                               char *var_name)
1590 {
1591         struct hist_trigger_data *var_hist_data;
1592         struct hist_var_data *var_data;
1593         struct trace_event_file *file, *found = NULL;
1594
1595         if (system)
1596                 return find_event_file(tr, system, event_name);
1597
1598         list_for_each_entry(var_data, &tr->hist_vars, list) {
1599                 var_hist_data = var_data->hist_data;
1600                 file = var_hist_data->event_file;
1601                 if (file == found)
1602                         continue;
1603
1604                 if (find_var_field(var_hist_data, var_name)) {
1605                         if (found) {
1606                                 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1607                                 return NULL;
1608                         }
1609
1610                         found = file;
1611                 }
1612         }
1613
1614         return found;
1615 }
1616
1617 static struct hist_field *find_file_var(struct trace_event_file *file,
1618                                         const char *var_name)
1619 {
1620         struct hist_trigger_data *test_data;
1621         struct event_trigger_data *test;
1622         struct hist_field *hist_field;
1623
1624         list_for_each_entry_rcu(test, &file->triggers, list) {
1625                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1626                         test_data = test->private_data;
1627                         hist_field = find_var_field(test_data, var_name);
1628                         if (hist_field)
1629                                 return hist_field;
1630                 }
1631         }
1632
1633         return NULL;
1634 }
1635
1636 static struct hist_field *
1637 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1638 {
1639         struct trace_array *tr = hist_data->event_file->tr;
1640         struct hist_field *hist_field, *found = NULL;
1641         struct trace_event_file *file;
1642         unsigned int i;
1643
1644         for (i = 0; i < hist_data->n_actions; i++) {
1645                 struct action_data *data = hist_data->actions[i];
1646
1647                 if (data->fn == action_trace) {
1648                         char *system = data->onmatch.match_event_system;
1649                         char *event_name = data->onmatch.match_event;
1650
1651                         file = find_var_file(tr, system, event_name, var_name);
1652                         if (!file)
1653                                 continue;
1654                         hist_field = find_file_var(file, var_name);
1655                         if (hist_field) {
1656                                 if (found) {
1657                                         hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1658                                         return ERR_PTR(-EINVAL);
1659                                 }
1660
1661                                 found = hist_field;
1662                         }
1663                 }
1664         }
1665         return found;
1666 }
1667
1668 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1669                                          char *system,
1670                                          char *event_name,
1671                                          char *var_name)
1672 {
1673         struct trace_array *tr = hist_data->event_file->tr;
1674         struct hist_field *hist_field = NULL;
1675         struct trace_event_file *file;
1676
1677         if (!system || !event_name) {
1678                 hist_field = find_match_var(hist_data, var_name);
1679                 if (IS_ERR(hist_field))
1680                         return NULL;
1681                 if (hist_field)
1682                         return hist_field;
1683         }
1684
1685         file = find_var_file(tr, system, event_name, var_name);
1686         if (!file)
1687                 return NULL;
1688
1689         hist_field = find_file_var(file, var_name);
1690
1691         return hist_field;
1692 }
1693
1694 struct hist_elt_data {
1695         char *comm;
1696         u64 *var_ref_vals;
1697         char *field_var_str[SYNTH_FIELDS_MAX];
1698 };
1699
1700 static u64 hist_field_var_ref(struct hist_field *hist_field,
1701                               struct tracing_map_elt *elt,
1702                               struct ring_buffer_event *rbe,
1703                               void *event)
1704 {
1705         struct hist_elt_data *elt_data;
1706         u64 var_val = 0;
1707
1708         elt_data = elt->private_data;
1709         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1710
1711         return var_val;
1712 }
1713
1714 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1715                              u64 *var_ref_vals, bool self)
1716 {
1717         struct hist_trigger_data *var_data;
1718         struct tracing_map_elt *var_elt;
1719         struct hist_field *hist_field;
1720         unsigned int i, var_idx;
1721         bool resolved = true;
1722         u64 var_val = 0;
1723
1724         for (i = 0; i < hist_data->n_var_refs; i++) {
1725                 hist_field = hist_data->var_refs[i];
1726                 var_idx = hist_field->var.idx;
1727                 var_data = hist_field->var.hist_data;
1728
1729                 if (var_data == NULL) {
1730                         resolved = false;
1731                         break;
1732                 }
1733
1734                 if ((self && var_data != hist_data) ||
1735                     (!self && var_data == hist_data))
1736                         continue;
1737
1738                 var_elt = tracing_map_lookup(var_data->map, key);
1739                 if (!var_elt) {
1740                         resolved = false;
1741                         break;
1742                 }
1743
1744                 if (!tracing_map_var_set(var_elt, var_idx)) {
1745                         resolved = false;
1746                         break;
1747                 }
1748
1749                 if (self || !hist_field->read_once)
1750                         var_val = tracing_map_read_var(var_elt, var_idx);
1751                 else
1752                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1753
1754                 var_ref_vals[i] = var_val;
1755         }
1756
1757         return resolved;
1758 }
1759
1760 static const char *hist_field_name(struct hist_field *field,
1761                                    unsigned int level)
1762 {
1763         const char *field_name = "";
1764
1765         if (level > 1)
1766                 return field_name;
1767
1768         if (field->field)
1769                 field_name = field->field->name;
1770         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1771                  field->flags & HIST_FIELD_FL_ALIAS)
1772                 field_name = hist_field_name(field->operands[0], ++level);
1773         else if (field->flags & HIST_FIELD_FL_CPU)
1774                 field_name = "cpu";
1775         else if (field->flags & HIST_FIELD_FL_EXPR ||
1776                  field->flags & HIST_FIELD_FL_VAR_REF) {
1777                 if (field->system) {
1778                         static char full_name[MAX_FILTER_STR_VAL];
1779
1780                         strcat(full_name, field->system);
1781                         strcat(full_name, ".");
1782                         strcat(full_name, field->event_name);
1783                         strcat(full_name, ".");
1784                         strcat(full_name, field->name);
1785                         field_name = full_name;
1786                 } else
1787                         field_name = field->name;
1788         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1789                 field_name = "common_timestamp";
1790
1791         if (field_name == NULL)
1792                 field_name = "";
1793
1794         return field_name;
1795 }
1796
1797 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1798 {
1799         hist_field_fn_t fn = NULL;
1800
1801         switch (field_size) {
1802         case 8:
1803                 if (field_is_signed)
1804                         fn = hist_field_s64;
1805                 else
1806                         fn = hist_field_u64;
1807                 break;
1808         case 4:
1809                 if (field_is_signed)
1810                         fn = hist_field_s32;
1811                 else
1812                         fn = hist_field_u32;
1813                 break;
1814         case 2:
1815                 if (field_is_signed)
1816                         fn = hist_field_s16;
1817                 else
1818                         fn = hist_field_u16;
1819                 break;
1820         case 1:
1821                 if (field_is_signed)
1822                         fn = hist_field_s8;
1823                 else
1824                         fn = hist_field_u8;
1825                 break;
1826         }
1827
1828         return fn;
1829 }
1830
1831 static int parse_map_size(char *str)
1832 {
1833         unsigned long size, map_bits;
1834         int ret;
1835
1836         strsep(&str, "=");
1837         if (!str) {
1838                 ret = -EINVAL;
1839                 goto out;
1840         }
1841
1842         ret = kstrtoul(str, 0, &size);
1843         if (ret)
1844                 goto out;
1845
1846         map_bits = ilog2(roundup_pow_of_two(size));
1847         if (map_bits < TRACING_MAP_BITS_MIN ||
1848             map_bits > TRACING_MAP_BITS_MAX)
1849                 ret = -EINVAL;
1850         else
1851                 ret = map_bits;
1852  out:
1853         return ret;
1854 }
1855
1856 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1857 {
1858         unsigned int i;
1859
1860         if (!attrs)
1861                 return;
1862
1863         for (i = 0; i < attrs->n_assignments; i++)
1864                 kfree(attrs->assignment_str[i]);
1865
1866         for (i = 0; i < attrs->n_actions; i++)
1867                 kfree(attrs->action_str[i]);
1868
1869         kfree(attrs->name);
1870         kfree(attrs->sort_key_str);
1871         kfree(attrs->keys_str);
1872         kfree(attrs->vals_str);
1873         kfree(attrs->clock);
1874         kfree(attrs);
1875 }
1876
1877 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1878 {
1879         int ret = -EINVAL;
1880
1881         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1882                 return ret;
1883
1884         if ((str_has_prefix(str, "onmatch(")) ||
1885             (str_has_prefix(str, "onmax("))) {
1886                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1887                 if (!attrs->action_str[attrs->n_actions]) {
1888                         ret = -ENOMEM;
1889                         return ret;
1890                 }
1891                 attrs->n_actions++;
1892                 ret = 0;
1893         }
1894
1895         return ret;
1896 }
1897
1898 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1899 {
1900         int ret = 0;
1901
1902         if ((str_has_prefix(str, "key=")) ||
1903             (str_has_prefix(str, "keys="))) {
1904                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
1905                 if (!attrs->keys_str) {
1906                         ret = -ENOMEM;
1907                         goto out;
1908                 }
1909         } else if ((str_has_prefix(str, "val=")) ||
1910                    (str_has_prefix(str, "vals=")) ||
1911                    (str_has_prefix(str, "values="))) {
1912                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
1913                 if (!attrs->vals_str) {
1914                         ret = -ENOMEM;
1915                         goto out;
1916                 }
1917         } else if (str_has_prefix(str, "sort=")) {
1918                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1919                 if (!attrs->sort_key_str) {
1920                         ret = -ENOMEM;
1921                         goto out;
1922                 }
1923         } else if (str_has_prefix(str, "name=")) {
1924                 attrs->name = kstrdup(str, GFP_KERNEL);
1925                 if (!attrs->name) {
1926                         ret = -ENOMEM;
1927                         goto out;
1928                 }
1929         } else if (str_has_prefix(str, "clock=")) {
1930                 strsep(&str, "=");
1931                 if (!str) {
1932                         ret = -EINVAL;
1933                         goto out;
1934                 }
1935
1936                 str = strstrip(str);
1937                 attrs->clock = kstrdup(str, GFP_KERNEL);
1938                 if (!attrs->clock) {
1939                         ret = -ENOMEM;
1940                         goto out;
1941                 }
1942         } else if (str_has_prefix(str, "size=")) {
1943                 int map_bits = parse_map_size(str);
1944
1945                 if (map_bits < 0) {
1946                         ret = map_bits;
1947                         goto out;
1948                 }
1949                 attrs->map_bits = map_bits;
1950         } else {
1951                 char *assignment;
1952
1953                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1954                         hist_err("Too many variables defined: ", str);
1955                         ret = -EINVAL;
1956                         goto out;
1957                 }
1958
1959                 assignment = kstrdup(str, GFP_KERNEL);
1960                 if (!assignment) {
1961                         ret = -ENOMEM;
1962                         goto out;
1963                 }
1964
1965                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1966         }
1967  out:
1968         return ret;
1969 }
1970
1971 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
1972 {
1973         struct hist_trigger_attrs *attrs;
1974         int ret = 0;
1975
1976         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1977         if (!attrs)
1978                 return ERR_PTR(-ENOMEM);
1979
1980         while (trigger_str) {
1981                 char *str = strsep(&trigger_str, ":");
1982
1983                 if (strchr(str, '=')) {
1984                         ret = parse_assignment(str, attrs);
1985                         if (ret)
1986                                 goto free;
1987                 } else if (strcmp(str, "pause") == 0)
1988                         attrs->pause = true;
1989                 else if ((strcmp(str, "cont") == 0) ||
1990                          (strcmp(str, "continue") == 0))
1991                         attrs->cont = true;
1992                 else if (strcmp(str, "clear") == 0)
1993                         attrs->clear = true;
1994                 else {
1995                         ret = parse_action(str, attrs);
1996                         if (ret)
1997                                 goto free;
1998                 }
1999         }
2000
2001         if (!attrs->keys_str) {
2002                 ret = -EINVAL;
2003                 goto free;
2004         }
2005
2006         if (!attrs->clock) {
2007                 attrs->clock = kstrdup("global", GFP_KERNEL);
2008                 if (!attrs->clock) {
2009                         ret = -ENOMEM;
2010                         goto free;
2011                 }
2012         }
2013
2014         return attrs;
2015  free:
2016         destroy_hist_trigger_attrs(attrs);
2017
2018         return ERR_PTR(ret);
2019 }
2020
2021 static inline void save_comm(char *comm, struct task_struct *task)
2022 {
2023         if (!task->pid) {
2024                 strcpy(comm, "<idle>");
2025                 return;
2026         }
2027
2028         if (WARN_ON_ONCE(task->pid < 0)) {
2029                 strcpy(comm, "<XXX>");
2030                 return;
2031         }
2032
2033         memcpy(comm, task->comm, TASK_COMM_LEN);
2034 }
2035
2036 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2037 {
2038         unsigned int i;
2039
2040         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2041                 kfree(elt_data->field_var_str[i]);
2042
2043         kfree(elt_data->comm);
2044         kfree(elt_data);
2045 }
2046
2047 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2048 {
2049         struct hist_elt_data *elt_data = elt->private_data;
2050
2051         hist_elt_data_free(elt_data);
2052 }
2053
2054 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2055 {
2056         struct hist_trigger_data *hist_data = elt->map->private_data;
2057         unsigned int size = TASK_COMM_LEN;
2058         struct hist_elt_data *elt_data;
2059         struct hist_field *key_field;
2060         unsigned int i, n_str;
2061
2062         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2063         if (!elt_data)
2064                 return -ENOMEM;
2065
2066         for_each_hist_key_field(i, hist_data) {
2067                 key_field = hist_data->fields[i];
2068
2069                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2070                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2071                         if (!elt_data->comm) {
2072                                 kfree(elt_data);
2073                                 return -ENOMEM;
2074                         }
2075                         break;
2076                 }
2077         }
2078
2079         n_str = hist_data->n_field_var_str + hist_data->n_max_var_str;
2080
2081         size = STR_VAR_LEN_MAX;
2082
2083         for (i = 0; i < n_str; i++) {
2084                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2085                 if (!elt_data->field_var_str[i]) {
2086                         hist_elt_data_free(elt_data);
2087                         return -ENOMEM;
2088                 }
2089         }
2090
2091         elt->private_data = elt_data;
2092
2093         return 0;
2094 }
2095
2096 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2097 {
2098         struct hist_elt_data *elt_data = elt->private_data;
2099
2100         if (elt_data->comm)
2101                 save_comm(elt_data->comm, current);
2102 }
2103
2104 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2105         .elt_alloc      = hist_trigger_elt_data_alloc,
2106         .elt_free       = hist_trigger_elt_data_free,
2107         .elt_init       = hist_trigger_elt_data_init,
2108 };
2109
2110 static const char *get_hist_field_flags(struct hist_field *hist_field)
2111 {
2112         const char *flags_str = NULL;
2113
2114         if (hist_field->flags & HIST_FIELD_FL_HEX)
2115                 flags_str = "hex";
2116         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2117                 flags_str = "sym";
2118         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2119                 flags_str = "sym-offset";
2120         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2121                 flags_str = "execname";
2122         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2123                 flags_str = "syscall";
2124         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2125                 flags_str = "log2";
2126         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2127                 flags_str = "usecs";
2128
2129         return flags_str;
2130 }
2131
2132 static void expr_field_str(struct hist_field *field, char *expr)
2133 {
2134         if (field->flags & HIST_FIELD_FL_VAR_REF)
2135                 strcat(expr, "$");
2136
2137         strcat(expr, hist_field_name(field, 0));
2138
2139         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2140                 const char *flags_str = get_hist_field_flags(field);
2141
2142                 if (flags_str) {
2143                         strcat(expr, ".");
2144                         strcat(expr, flags_str);
2145                 }
2146         }
2147 }
2148
2149 static char *expr_str(struct hist_field *field, unsigned int level)
2150 {
2151         char *expr;
2152
2153         if (level > 1)
2154                 return NULL;
2155
2156         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2157         if (!expr)
2158                 return NULL;
2159
2160         if (!field->operands[0]) {
2161                 expr_field_str(field, expr);
2162                 return expr;
2163         }
2164
2165         if (field->operator == FIELD_OP_UNARY_MINUS) {
2166                 char *subexpr;
2167
2168                 strcat(expr, "-(");
2169                 subexpr = expr_str(field->operands[0], ++level);
2170                 if (!subexpr) {
2171                         kfree(expr);
2172                         return NULL;
2173                 }
2174                 strcat(expr, subexpr);
2175                 strcat(expr, ")");
2176
2177                 kfree(subexpr);
2178
2179                 return expr;
2180         }
2181
2182         expr_field_str(field->operands[0], expr);
2183
2184         switch (field->operator) {
2185         case FIELD_OP_MINUS:
2186                 strcat(expr, "-");
2187                 break;
2188         case FIELD_OP_PLUS:
2189                 strcat(expr, "+");
2190                 break;
2191         default:
2192                 kfree(expr);
2193                 return NULL;
2194         }
2195
2196         expr_field_str(field->operands[1], expr);
2197
2198         return expr;
2199 }
2200
2201 static int contains_operator(char *str)
2202 {
2203         enum field_op_id field_op = FIELD_OP_NONE;
2204         char *op;
2205
2206         op = strpbrk(str, "+-");
2207         if (!op)
2208                 return FIELD_OP_NONE;
2209
2210         switch (*op) {
2211         case '-':
2212                 if (*str == '-')
2213                         field_op = FIELD_OP_UNARY_MINUS;
2214                 else
2215                         field_op = FIELD_OP_MINUS;
2216                 break;
2217         case '+':
2218                 field_op = FIELD_OP_PLUS;
2219                 break;
2220         default:
2221                 break;
2222         }
2223
2224         return field_op;
2225 }
2226
2227 static void __destroy_hist_field(struct hist_field *hist_field)
2228 {
2229         kfree(hist_field->var.name);
2230         kfree(hist_field->name);
2231         kfree(hist_field->type);
2232
2233         kfree(hist_field);
2234 }
2235
2236 static void destroy_hist_field(struct hist_field *hist_field,
2237                                unsigned int level)
2238 {
2239         unsigned int i;
2240
2241         if (level > 3)
2242                 return;
2243
2244         if (!hist_field)
2245                 return;
2246
2247         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2248                 return; /* var refs will be destroyed separately */
2249
2250         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2251                 destroy_hist_field(hist_field->operands[i], level + 1);
2252
2253         __destroy_hist_field(hist_field);
2254 }
2255
2256 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2257                                             struct ftrace_event_field *field,
2258                                             unsigned long flags,
2259                                             char *var_name)
2260 {
2261         struct hist_field *hist_field;
2262
2263         if (field && is_function_field(field))
2264                 return NULL;
2265
2266         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2267         if (!hist_field)
2268                 return NULL;
2269
2270         hist_field->hist_data = hist_data;
2271
2272         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2273                 goto out; /* caller will populate */
2274
2275         if (flags & HIST_FIELD_FL_VAR_REF) {
2276                 hist_field->fn = hist_field_var_ref;
2277                 goto out;
2278         }
2279
2280         if (flags & HIST_FIELD_FL_HITCOUNT) {
2281                 hist_field->fn = hist_field_counter;
2282                 hist_field->size = sizeof(u64);
2283                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2284                 if (!hist_field->type)
2285                         goto free;
2286                 goto out;
2287         }
2288
2289         if (flags & HIST_FIELD_FL_STACKTRACE) {
2290                 hist_field->fn = hist_field_none;
2291                 goto out;
2292         }
2293
2294         if (flags & HIST_FIELD_FL_LOG2) {
2295                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2296                 hist_field->fn = hist_field_log2;
2297                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2298                 hist_field->size = hist_field->operands[0]->size;
2299                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2300                 if (!hist_field->type)
2301                         goto free;
2302                 goto out;
2303         }
2304
2305         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2306                 hist_field->fn = hist_field_timestamp;
2307                 hist_field->size = sizeof(u64);
2308                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2309                 if (!hist_field->type)
2310                         goto free;
2311                 goto out;
2312         }
2313
2314         if (flags & HIST_FIELD_FL_CPU) {
2315                 hist_field->fn = hist_field_cpu;
2316                 hist_field->size = sizeof(int);
2317                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2318                 if (!hist_field->type)
2319                         goto free;
2320                 goto out;
2321         }
2322
2323         if (WARN_ON_ONCE(!field))
2324                 goto out;
2325
2326         if (is_string_field(field)) {
2327                 flags |= HIST_FIELD_FL_STRING;
2328
2329                 hist_field->size = MAX_FILTER_STR_VAL;
2330                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2331                 if (!hist_field->type)
2332                         goto free;
2333
2334                 if (field->filter_type == FILTER_STATIC_STRING)
2335                         hist_field->fn = hist_field_string;
2336                 else if (field->filter_type == FILTER_DYN_STRING)
2337                         hist_field->fn = hist_field_dynstring;
2338                 else
2339                         hist_field->fn = hist_field_pstring;
2340         } else {
2341                 hist_field->size = field->size;
2342                 hist_field->is_signed = field->is_signed;
2343                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2344                 if (!hist_field->type)
2345                         goto free;
2346
2347                 hist_field->fn = select_value_fn(field->size,
2348                                                  field->is_signed);
2349                 if (!hist_field->fn) {
2350                         destroy_hist_field(hist_field, 0);
2351                         return NULL;
2352                 }
2353         }
2354  out:
2355         hist_field->field = field;
2356         hist_field->flags = flags;
2357
2358         if (var_name) {
2359                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2360                 if (!hist_field->var.name)
2361                         goto free;
2362         }
2363
2364         return hist_field;
2365  free:
2366         destroy_hist_field(hist_field, 0);
2367         return NULL;
2368 }
2369
2370 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2371 {
2372         unsigned int i;
2373
2374         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2375                 if (hist_data->fields[i]) {
2376                         destroy_hist_field(hist_data->fields[i], 0);
2377                         hist_data->fields[i] = NULL;
2378                 }
2379         }
2380
2381         for (i = 0; i < hist_data->n_var_refs; i++) {
2382                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2383                 __destroy_hist_field(hist_data->var_refs[i]);
2384                 hist_data->var_refs[i] = NULL;
2385         }
2386 }
2387
2388 static int init_var_ref(struct hist_field *ref_field,
2389                         struct hist_field *var_field,
2390                         char *system, char *event_name)
2391 {
2392         int err = 0;
2393
2394         ref_field->var.idx = var_field->var.idx;
2395         ref_field->var.hist_data = var_field->hist_data;
2396         ref_field->size = var_field->size;
2397         ref_field->is_signed = var_field->is_signed;
2398         ref_field->flags |= var_field->flags &
2399                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2400
2401         if (system) {
2402                 ref_field->system = kstrdup(system, GFP_KERNEL);
2403                 if (!ref_field->system)
2404                         return -ENOMEM;
2405         }
2406
2407         if (event_name) {
2408                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2409                 if (!ref_field->event_name) {
2410                         err = -ENOMEM;
2411                         goto free;
2412                 }
2413         }
2414
2415         if (var_field->var.name) {
2416                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2417                 if (!ref_field->name) {
2418                         err = -ENOMEM;
2419                         goto free;
2420                 }
2421         } else if (var_field->name) {
2422                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2423                 if (!ref_field->name) {
2424                         err = -ENOMEM;
2425                         goto free;
2426                 }
2427         }
2428
2429         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2430         if (!ref_field->type) {
2431                 err = -ENOMEM;
2432                 goto free;
2433         }
2434  out:
2435         return err;
2436  free:
2437         kfree(ref_field->system);
2438         kfree(ref_field->event_name);
2439         kfree(ref_field->name);
2440
2441         goto out;
2442 }
2443
2444 /**
2445  * create_var_ref - Create a variable reference and attach it to trigger
2446  * @hist_data: The trigger that will be referencing the variable
2447  * @var_field: The VAR field to create a reference to
2448  * @system: The optional system string
2449  * @event_name: The optional event_name string
2450  *
2451  * Given a variable hist_field, create a VAR_REF hist_field that
2452  * represents a reference to it.
2453  *
2454  * This function also adds the reference to the trigger that
2455  * now references the variable.
2456  *
2457  * Return: The VAR_REF field if successful, NULL if not
2458  */
2459 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2460                                          struct hist_field *var_field,
2461                                          char *system, char *event_name)
2462 {
2463         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2464         struct hist_field *ref_field;
2465
2466         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2467         if (ref_field) {
2468                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2469                         destroy_hist_field(ref_field, 0);
2470                         return NULL;
2471                 }
2472
2473                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2474                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2475         }
2476
2477         return ref_field;
2478 }
2479
2480 static bool is_var_ref(char *var_name)
2481 {
2482         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2483                 return false;
2484
2485         return true;
2486 }
2487
2488 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2489                                  char *var_name)
2490 {
2491         char *name, *field;
2492         unsigned int i;
2493
2494         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2495                 name = hist_data->attrs->var_defs.name[i];
2496
2497                 if (strcmp(var_name, name) == 0) {
2498                         field = hist_data->attrs->var_defs.expr[i];
2499                         if (contains_operator(field) || is_var_ref(field))
2500                                 continue;
2501                         return field;
2502                 }
2503         }
2504
2505         return NULL;
2506 }
2507
2508 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2509                                  char *system, char *event_name,
2510                                  char *var_name)
2511 {
2512         struct trace_event_call *call;
2513
2514         if (system && event_name) {
2515                 call = hist_data->event_file->event_call;
2516
2517                 if (strcmp(system, call->class->system) != 0)
2518                         return NULL;
2519
2520                 if (strcmp(event_name, trace_event_name(call)) != 0)
2521                         return NULL;
2522         }
2523
2524         if (!!system != !!event_name)
2525                 return NULL;
2526
2527         if (!is_var_ref(var_name))
2528                 return NULL;
2529
2530         var_name++;
2531
2532         return field_name_from_var(hist_data, var_name);
2533 }
2534
2535 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2536                                         char *system, char *event_name,
2537                                         char *var_name)
2538 {
2539         struct hist_field *var_field = NULL, *ref_field = NULL;
2540
2541         if (!is_var_ref(var_name))
2542                 return NULL;
2543
2544         var_name++;
2545
2546         var_field = find_event_var(hist_data, system, event_name, var_name);
2547         if (var_field)
2548                 ref_field = create_var_ref(hist_data, var_field,
2549                                            system, event_name);
2550
2551         if (!ref_field)
2552                 hist_err_event("Couldn't find variable: $",
2553                                system, event_name, var_name);
2554
2555         return ref_field;
2556 }
2557
2558 static struct ftrace_event_field *
2559 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2560             char *field_str, unsigned long *flags)
2561 {
2562         struct ftrace_event_field *field = NULL;
2563         char *field_name, *modifier, *str;
2564
2565         modifier = str = kstrdup(field_str, GFP_KERNEL);
2566         if (!modifier)
2567                 return ERR_PTR(-ENOMEM);
2568
2569         field_name = strsep(&modifier, ".");
2570         if (modifier) {
2571                 if (strcmp(modifier, "hex") == 0)
2572                         *flags |= HIST_FIELD_FL_HEX;
2573                 else if (strcmp(modifier, "sym") == 0)
2574                         *flags |= HIST_FIELD_FL_SYM;
2575                 else if (strcmp(modifier, "sym-offset") == 0)
2576                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2577                 else if ((strcmp(modifier, "execname") == 0) &&
2578                          (strcmp(field_name, "common_pid") == 0))
2579                         *flags |= HIST_FIELD_FL_EXECNAME;
2580                 else if (strcmp(modifier, "syscall") == 0)
2581                         *flags |= HIST_FIELD_FL_SYSCALL;
2582                 else if (strcmp(modifier, "log2") == 0)
2583                         *flags |= HIST_FIELD_FL_LOG2;
2584                 else if (strcmp(modifier, "usecs") == 0)
2585                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2586                 else {
2587                         hist_err("Invalid field modifier: ", modifier);
2588                         field = ERR_PTR(-EINVAL);
2589                         goto out;
2590                 }
2591         }
2592
2593         if (strcmp(field_name, "common_timestamp") == 0) {
2594                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2595                 hist_data->enable_timestamps = true;
2596                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2597                         hist_data->attrs->ts_in_usecs = true;
2598         } else if (strcmp(field_name, "cpu") == 0)
2599                 *flags |= HIST_FIELD_FL_CPU;
2600         else {
2601                 field = trace_find_event_field(file->event_call, field_name);
2602                 if (!field || !field->size) {
2603                         hist_err("Couldn't find field: ", field_name);
2604                         field = ERR_PTR(-EINVAL);
2605                         goto out;
2606                 }
2607         }
2608  out:
2609         kfree(str);
2610
2611         return field;
2612 }
2613
2614 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2615                                        struct hist_field *var_ref,
2616                                        char *var_name)
2617 {
2618         struct hist_field *alias = NULL;
2619         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2620
2621         alias = create_hist_field(hist_data, NULL, flags, var_name);
2622         if (!alias)
2623                 return NULL;
2624
2625         alias->fn = var_ref->fn;
2626         alias->operands[0] = var_ref;
2627
2628         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2629                 destroy_hist_field(alias, 0);
2630                 return NULL;
2631         }
2632
2633         return alias;
2634 }
2635
2636 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2637                                      struct trace_event_file *file, char *str,
2638                                      unsigned long *flags, char *var_name)
2639 {
2640         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2641         struct ftrace_event_field *field = NULL;
2642         struct hist_field *hist_field = NULL;
2643         int ret = 0;
2644
2645         s = strchr(str, '.');
2646         if (s) {
2647                 s = strchr(++s, '.');
2648                 if (s) {
2649                         ref_system = strsep(&str, ".");
2650                         if (!str) {
2651                                 ret = -EINVAL;
2652                                 goto out;
2653                         }
2654                         ref_event = strsep(&str, ".");
2655                         if (!str) {
2656                                 ret = -EINVAL;
2657                                 goto out;
2658                         }
2659                         ref_var = str;
2660                 }
2661         }
2662
2663         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2664         if (!s) {
2665                 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2666                 if (hist_field) {
2667                         if (var_name) {
2668                                 hist_field = create_alias(hist_data, hist_field, var_name);
2669                                 if (!hist_field) {
2670                                         ret = -ENOMEM;
2671                                         goto out;
2672                                 }
2673                         }
2674                         return hist_field;
2675                 }
2676         } else
2677                 str = s;
2678
2679         field = parse_field(hist_data, file, str, flags);
2680         if (IS_ERR(field)) {
2681                 ret = PTR_ERR(field);
2682                 goto out;
2683         }
2684
2685         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2686         if (!hist_field) {
2687                 ret = -ENOMEM;
2688                 goto out;
2689         }
2690
2691         return hist_field;
2692  out:
2693         return ERR_PTR(ret);
2694 }
2695
2696 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2697                                      struct trace_event_file *file,
2698                                      char *str, unsigned long flags,
2699                                      char *var_name, unsigned int level);
2700
2701 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2702                                       struct trace_event_file *file,
2703                                       char *str, unsigned long flags,
2704                                       char *var_name, unsigned int level)
2705 {
2706         struct hist_field *operand1, *expr = NULL;
2707         unsigned long operand_flags;
2708         int ret = 0;
2709         char *s;
2710
2711         /* we support only -(xxx) i.e. explicit parens required */
2712
2713         if (level > 3) {
2714                 hist_err("Too many subexpressions (3 max): ", str);
2715                 ret = -EINVAL;
2716                 goto free;
2717         }
2718
2719         str++; /* skip leading '-' */
2720
2721         s = strchr(str, '(');
2722         if (s)
2723                 str++;
2724         else {
2725                 ret = -EINVAL;
2726                 goto free;
2727         }
2728
2729         s = strrchr(str, ')');
2730         if (s)
2731                 *s = '\0';
2732         else {
2733                 ret = -EINVAL; /* no closing ')' */
2734                 goto free;
2735         }
2736
2737         flags |= HIST_FIELD_FL_EXPR;
2738         expr = create_hist_field(hist_data, NULL, flags, var_name);
2739         if (!expr) {
2740                 ret = -ENOMEM;
2741                 goto free;
2742         }
2743
2744         operand_flags = 0;
2745         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2746         if (IS_ERR(operand1)) {
2747                 ret = PTR_ERR(operand1);
2748                 goto free;
2749         }
2750
2751         expr->flags |= operand1->flags &
2752                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2753         expr->fn = hist_field_unary_minus;
2754         expr->operands[0] = operand1;
2755         expr->operator = FIELD_OP_UNARY_MINUS;
2756         expr->name = expr_str(expr, 0);
2757         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2758         if (!expr->type) {
2759                 ret = -ENOMEM;
2760                 goto free;
2761         }
2762
2763         return expr;
2764  free:
2765         destroy_hist_field(expr, 0);
2766         return ERR_PTR(ret);
2767 }
2768
2769 static int check_expr_operands(struct hist_field *operand1,
2770                                struct hist_field *operand2)
2771 {
2772         unsigned long operand1_flags = operand1->flags;
2773         unsigned long operand2_flags = operand2->flags;
2774
2775         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2776             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2777                 struct hist_field *var;
2778
2779                 var = find_var_field(operand1->var.hist_data, operand1->name);
2780                 if (!var)
2781                         return -EINVAL;
2782                 operand1_flags = var->flags;
2783         }
2784
2785         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2786             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2787                 struct hist_field *var;
2788
2789                 var = find_var_field(operand2->var.hist_data, operand2->name);
2790                 if (!var)
2791                         return -EINVAL;
2792                 operand2_flags = var->flags;
2793         }
2794
2795         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2796             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2797                 hist_err("Timestamp units in expression don't match", NULL);
2798                 return -EINVAL;
2799         }
2800
2801         return 0;
2802 }
2803
2804 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2805                                      struct trace_event_file *file,
2806                                      char *str, unsigned long flags,
2807                                      char *var_name, unsigned int level)
2808 {
2809         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2810         unsigned long operand_flags;
2811         int field_op, ret = -EINVAL;
2812         char *sep, *operand1_str;
2813
2814         if (level > 3) {
2815                 hist_err("Too many subexpressions (3 max): ", str);
2816                 return ERR_PTR(-EINVAL);
2817         }
2818
2819         field_op = contains_operator(str);
2820
2821         if (field_op == FIELD_OP_NONE)
2822                 return parse_atom(hist_data, file, str, &flags, var_name);
2823
2824         if (field_op == FIELD_OP_UNARY_MINUS)
2825                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2826
2827         switch (field_op) {
2828         case FIELD_OP_MINUS:
2829                 sep = "-";
2830                 break;
2831         case FIELD_OP_PLUS:
2832                 sep = "+";
2833                 break;
2834         default:
2835                 goto free;
2836         }
2837
2838         operand1_str = strsep(&str, sep);
2839         if (!operand1_str || !str)
2840                 goto free;
2841
2842         operand_flags = 0;
2843         operand1 = parse_atom(hist_data, file, operand1_str,
2844                               &operand_flags, NULL);
2845         if (IS_ERR(operand1)) {
2846                 ret = PTR_ERR(operand1);
2847                 operand1 = NULL;
2848                 goto free;
2849         }
2850
2851         /* rest of string could be another expression e.g. b+c in a+b+c */
2852         operand_flags = 0;
2853         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2854         if (IS_ERR(operand2)) {
2855                 ret = PTR_ERR(operand2);
2856                 operand2 = NULL;
2857                 goto free;
2858         }
2859
2860         ret = check_expr_operands(operand1, operand2);
2861         if (ret)
2862                 goto free;
2863
2864         flags |= HIST_FIELD_FL_EXPR;
2865
2866         flags |= operand1->flags &
2867                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2868
2869         expr = create_hist_field(hist_data, NULL, flags, var_name);
2870         if (!expr) {
2871                 ret = -ENOMEM;
2872                 goto free;
2873         }
2874
2875         operand1->read_once = true;
2876         operand2->read_once = true;
2877
2878         expr->operands[0] = operand1;
2879         expr->operands[1] = operand2;
2880         expr->operator = field_op;
2881         expr->name = expr_str(expr, 0);
2882         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2883         if (!expr->type) {
2884                 ret = -ENOMEM;
2885                 goto free;
2886         }
2887
2888         switch (field_op) {
2889         case FIELD_OP_MINUS:
2890                 expr->fn = hist_field_minus;
2891                 break;
2892         case FIELD_OP_PLUS:
2893                 expr->fn = hist_field_plus;
2894                 break;
2895         default:
2896                 ret = -EINVAL;
2897                 goto free;
2898         }
2899
2900         return expr;
2901  free:
2902         destroy_hist_field(operand1, 0);
2903         destroy_hist_field(operand2, 0);
2904         destroy_hist_field(expr, 0);
2905
2906         return ERR_PTR(ret);
2907 }
2908
2909 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2910                                  struct trace_event_file *file)
2911 {
2912         struct event_trigger_data *test;
2913
2914         list_for_each_entry_rcu(test, &file->triggers, list) {
2915                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2916                         if (test->private_data == hist_data)
2917                                 return test->filter_str;
2918                 }
2919         }
2920
2921         return NULL;
2922 }
2923
2924 static struct event_command trigger_hist_cmd;
2925 static int event_hist_trigger_func(struct event_command *cmd_ops,
2926                                    struct trace_event_file *file,
2927                                    char *glob, char *cmd, char *param);
2928
2929 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2930                             struct hist_trigger_data *hist_data,
2931                             unsigned int n_keys)
2932 {
2933         struct hist_field *target_hist_field, *hist_field;
2934         unsigned int n, i, j;
2935
2936         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2937                 return false;
2938
2939         i = hist_data->n_vals;
2940         j = target_hist_data->n_vals;
2941
2942         for (n = 0; n < n_keys; n++) {
2943                 hist_field = hist_data->fields[i + n];
2944                 target_hist_field = target_hist_data->fields[j + n];
2945
2946                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2947                         return false;
2948                 if (hist_field->size != target_hist_field->size)
2949                         return false;
2950                 if (hist_field->is_signed != target_hist_field->is_signed)
2951                         return false;
2952         }
2953
2954         return true;
2955 }
2956
2957 static struct hist_trigger_data *
2958 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2959                      struct trace_event_file *file)
2960 {
2961         struct hist_trigger_data *hist_data;
2962         struct event_trigger_data *test;
2963         unsigned int n_keys;
2964
2965         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2966
2967         list_for_each_entry_rcu(test, &file->triggers, list) {
2968                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2969                         hist_data = test->private_data;
2970
2971                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2972                                 return hist_data;
2973                 }
2974         }
2975
2976         return NULL;
2977 }
2978
2979 static struct trace_event_file *event_file(struct trace_array *tr,
2980                                            char *system, char *event_name)
2981 {
2982         struct trace_event_file *file;
2983
2984         file = __find_event_file(tr, system, event_name);
2985         if (!file)
2986                 return ERR_PTR(-EINVAL);
2987
2988         return file;
2989 }
2990
2991 static struct hist_field *
2992 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2993                          char *system, char *event_name, char *field_name)
2994 {
2995         struct hist_field *event_var;
2996         char *synthetic_name;
2997
2998         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2999         if (!synthetic_name)
3000                 return ERR_PTR(-ENOMEM);
3001
3002         strcpy(synthetic_name, "synthetic_");
3003         strcat(synthetic_name, field_name);
3004
3005         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3006
3007         kfree(synthetic_name);
3008
3009         return event_var;
3010 }
3011
3012 /**
3013  * create_field_var_hist - Automatically create a histogram and var for a field
3014  * @target_hist_data: The target hist trigger
3015  * @subsys_name: Optional subsystem name
3016  * @event_name: Optional event name
3017  * @field_name: The name of the field (and the resulting variable)
3018  *
3019  * Hist trigger actions fetch data from variables, not directly from
3020  * events.  However, for convenience, users are allowed to directly
3021  * specify an event field in an action, which will be automatically
3022  * converted into a variable on their behalf.
3023
3024  * If a user specifies a field on an event that isn't the event the
3025  * histogram currently being defined (the target event histogram), the
3026  * only way that can be accomplished is if a new hist trigger is
3027  * created and the field variable defined on that.
3028  *
3029  * This function creates a new histogram compatible with the target
3030  * event (meaning a histogram with the same key as the target
3031  * histogram), and creates a variable for the specified field, but
3032  * with 'synthetic_' prepended to the variable name in order to avoid
3033  * collision with normal field variables.
3034  *
3035  * Return: The variable created for the field.
3036  */
3037 static struct hist_field *
3038 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3039                       char *subsys_name, char *event_name, char *field_name)
3040 {
3041         struct trace_array *tr = target_hist_data->event_file->tr;
3042         struct hist_field *event_var = ERR_PTR(-EINVAL);
3043         struct hist_trigger_data *hist_data;
3044         unsigned int i, n, first = true;
3045         struct field_var_hist *var_hist;
3046         struct trace_event_file *file;
3047         struct hist_field *key_field;
3048         char *saved_filter;
3049         char *cmd;
3050         int ret;
3051
3052         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3053                 hist_err_event("onmatch: Too many field variables defined: ",
3054                                subsys_name, event_name, field_name);
3055                 return ERR_PTR(-EINVAL);
3056         }
3057
3058         file = event_file(tr, subsys_name, event_name);
3059
3060         if (IS_ERR(file)) {
3061                 hist_err_event("onmatch: Event file not found: ",
3062                                subsys_name, event_name, field_name);
3063                 ret = PTR_ERR(file);
3064                 return ERR_PTR(ret);
3065         }
3066
3067         /*
3068          * Look for a histogram compatible with target.  We'll use the
3069          * found histogram specification to create a new matching
3070          * histogram with our variable on it.  target_hist_data is not
3071          * yet a registered histogram so we can't use that.
3072          */
3073         hist_data = find_compatible_hist(target_hist_data, file);
3074         if (!hist_data) {
3075                 hist_err_event("onmatch: Matching event histogram not found: ",
3076                                subsys_name, event_name, field_name);
3077                 return ERR_PTR(-EINVAL);
3078         }
3079
3080         /* See if a synthetic field variable has already been created */
3081         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3082                                              event_name, field_name);
3083         if (!IS_ERR_OR_NULL(event_var))
3084                 return event_var;
3085
3086         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3087         if (!var_hist)
3088                 return ERR_PTR(-ENOMEM);
3089
3090         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3091         if (!cmd) {
3092                 kfree(var_hist);
3093                 return ERR_PTR(-ENOMEM);
3094         }
3095
3096         /* Use the same keys as the compatible histogram */
3097         strcat(cmd, "keys=");
3098
3099         for_each_hist_key_field(i, hist_data) {
3100                 key_field = hist_data->fields[i];
3101                 if (!first)
3102                         strcat(cmd, ",");
3103                 strcat(cmd, key_field->field->name);
3104                 first = false;
3105         }
3106
3107         /* Create the synthetic field variable specification */
3108         strcat(cmd, ":synthetic_");
3109         strcat(cmd, field_name);
3110         strcat(cmd, "=");
3111         strcat(cmd, field_name);
3112
3113         /* Use the same filter as the compatible histogram */
3114         saved_filter = find_trigger_filter(hist_data, file);
3115         if (saved_filter) {
3116                 strcat(cmd, " if ");
3117                 strcat(cmd, saved_filter);
3118         }
3119
3120         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3121         if (!var_hist->cmd) {
3122                 kfree(cmd);
3123                 kfree(var_hist);
3124                 return ERR_PTR(-ENOMEM);
3125         }
3126
3127         /* Save the compatible histogram information */
3128         var_hist->hist_data = hist_data;
3129
3130         /* Create the new histogram with our variable */
3131         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3132                                       "", "hist", cmd);
3133         if (ret) {
3134                 kfree(cmd);
3135                 kfree(var_hist->cmd);
3136                 kfree(var_hist);
3137                 hist_err_event("onmatch: Couldn't create histogram for field: ",
3138                                subsys_name, event_name, field_name);
3139                 return ERR_PTR(ret);
3140         }
3141
3142         kfree(cmd);
3143
3144         /* If we can't find the variable, something went wrong */
3145         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3146                                              event_name, field_name);
3147         if (IS_ERR_OR_NULL(event_var)) {
3148                 kfree(var_hist->cmd);
3149                 kfree(var_hist);
3150                 hist_err_event("onmatch: Couldn't find synthetic variable: ",
3151                                subsys_name, event_name, field_name);
3152                 return ERR_PTR(-EINVAL);
3153         }
3154
3155         n = target_hist_data->n_field_var_hists;
3156         target_hist_data->field_var_hists[n] = var_hist;
3157         target_hist_data->n_field_var_hists++;
3158
3159         return event_var;
3160 }
3161
3162 static struct hist_field *
3163 find_target_event_var(struct hist_trigger_data *hist_data,
3164                       char *subsys_name, char *event_name, char *var_name)
3165 {
3166         struct trace_event_file *file = hist_data->event_file;
3167         struct hist_field *hist_field = NULL;
3168
3169         if (subsys_name) {
3170                 struct trace_event_call *call;
3171
3172                 if (!event_name)
3173                         return NULL;
3174
3175                 call = file->event_call;
3176
3177                 if (strcmp(subsys_name, call->class->system) != 0)
3178                         return NULL;
3179
3180                 if (strcmp(event_name, trace_event_name(call)) != 0)
3181                         return NULL;
3182         }
3183
3184         hist_field = find_var_field(hist_data, var_name);
3185
3186         return hist_field;
3187 }
3188
3189 static inline void __update_field_vars(struct tracing_map_elt *elt,
3190                                        struct ring_buffer_event *rbe,
3191                                        void *rec,
3192                                        struct field_var **field_vars,
3193                                        unsigned int n_field_vars,
3194                                        unsigned int field_var_str_start)
3195 {
3196         struct hist_elt_data *elt_data = elt->private_data;
3197         unsigned int i, j, var_idx;
3198         u64 var_val;
3199
3200         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3201                 struct field_var *field_var = field_vars[i];
3202                 struct hist_field *var = field_var->var;
3203                 struct hist_field *val = field_var->val;
3204
3205                 var_val = val->fn(val, elt, rbe, rec);
3206                 var_idx = var->var.idx;
3207
3208                 if (val->flags & HIST_FIELD_FL_STRING) {
3209                         char *str = elt_data->field_var_str[j++];
3210                         char *val_str = (char *)(uintptr_t)var_val;
3211
3212                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3213                         var_val = (u64)(uintptr_t)str;
3214                 }
3215                 tracing_map_set_var(elt, var_idx, var_val);
3216         }
3217 }
3218
3219 static void update_field_vars(struct hist_trigger_data *hist_data,
3220                               struct tracing_map_elt *elt,
3221                               struct ring_buffer_event *rbe,
3222                               void *rec)
3223 {
3224         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3225                             hist_data->n_field_vars, 0);
3226 }
3227
3228 static void update_max_vars(struct hist_trigger_data *hist_data,
3229                             struct tracing_map_elt *elt,
3230                             struct ring_buffer_event *rbe,
3231                             void *rec)
3232 {
3233         __update_field_vars(elt, rbe, rec, hist_data->max_vars,
3234                             hist_data->n_max_vars, hist_data->n_field_var_str);
3235 }
3236
3237 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3238                                      struct trace_event_file *file,
3239                                      char *name, int size, const char *type)
3240 {
3241         struct hist_field *var;
3242         int idx;
3243
3244         if (find_var(hist_data, file, name) && !hist_data->remove) {
3245                 var = ERR_PTR(-EINVAL);
3246                 goto out;
3247         }
3248
3249         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3250         if (!var) {
3251                 var = ERR_PTR(-ENOMEM);
3252                 goto out;
3253         }
3254
3255         idx = tracing_map_add_var(hist_data->map);
3256         if (idx < 0) {
3257                 kfree(var);
3258                 var = ERR_PTR(-EINVAL);
3259                 goto out;
3260         }
3261
3262         var->flags = HIST_FIELD_FL_VAR;
3263         var->var.idx = idx;
3264         var->var.hist_data = var->hist_data = hist_data;
3265         var->size = size;
3266         var->var.name = kstrdup(name, GFP_KERNEL);
3267         var->type = kstrdup(type, GFP_KERNEL);
3268         if (!var->var.name || !var->type) {
3269                 kfree(var->var.name);
3270                 kfree(var->type);
3271                 kfree(var);
3272                 var = ERR_PTR(-ENOMEM);
3273         }
3274  out:
3275         return var;
3276 }
3277
3278 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3279                                           struct trace_event_file *file,
3280                                           char *field_name)
3281 {
3282         struct hist_field *val = NULL, *var = NULL;
3283         unsigned long flags = HIST_FIELD_FL_VAR;
3284         struct field_var *field_var;
3285         int ret = 0;
3286
3287         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3288                 hist_err("Too many field variables defined: ", field_name);
3289                 ret = -EINVAL;
3290                 goto err;
3291         }
3292
3293         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3294         if (IS_ERR(val)) {
3295                 hist_err("Couldn't parse field variable: ", field_name);
3296                 ret = PTR_ERR(val);
3297                 goto err;
3298         }
3299
3300         var = create_var(hist_data, file, field_name, val->size, val->type);
3301         if (IS_ERR(var)) {
3302                 hist_err("Couldn't create or find variable: ", field_name);
3303                 kfree(val);
3304                 ret = PTR_ERR(var);
3305                 goto err;
3306         }
3307
3308         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3309         if (!field_var) {
3310                 kfree(val);
3311                 kfree(var);
3312                 ret =  -ENOMEM;
3313                 goto err;
3314         }
3315
3316         field_var->var = var;
3317         field_var->val = val;
3318  out:
3319         return field_var;
3320  err:
3321         field_var = ERR_PTR(ret);
3322         goto out;
3323 }
3324
3325 /**
3326  * create_target_field_var - Automatically create a variable for a field
3327  * @target_hist_data: The target hist trigger
3328  * @subsys_name: Optional subsystem name
3329  * @event_name: Optional event name
3330  * @var_name: The name of the field (and the resulting variable)
3331  *
3332  * Hist trigger actions fetch data from variables, not directly from
3333  * events.  However, for convenience, users are allowed to directly
3334  * specify an event field in an action, which will be automatically
3335  * converted into a variable on their behalf.
3336
3337  * This function creates a field variable with the name var_name on
3338  * the hist trigger currently being defined on the target event.  If
3339  * subsys_name and event_name are specified, this function simply
3340  * verifies that they do in fact match the target event subsystem and
3341  * event name.
3342  *
3343  * Return: The variable created for the field.
3344  */
3345 static struct field_var *
3346 create_target_field_var(struct hist_trigger_data *target_hist_data,
3347                         char *subsys_name, char *event_name, char *var_name)
3348 {
3349         struct trace_event_file *file = target_hist_data->event_file;
3350
3351         if (subsys_name) {
3352                 struct trace_event_call *call;
3353
3354                 if (!event_name)
3355                         return NULL;
3356
3357                 call = file->event_call;
3358
3359                 if (strcmp(subsys_name, call->class->system) != 0)
3360                         return NULL;
3361
3362                 if (strcmp(event_name, trace_event_name(call)) != 0)
3363                         return NULL;
3364         }
3365
3366         return create_field_var(target_hist_data, file, var_name);
3367 }
3368
3369 static void onmax_print(struct seq_file *m,
3370                         struct hist_trigger_data *hist_data,
3371                         struct tracing_map_elt *elt,
3372                         struct action_data *data)
3373 {
3374         unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx;
3375
3376         seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx));
3377
3378         for (i = 0; i < hist_data->n_max_vars; i++) {
3379                 struct hist_field *save_val = hist_data->max_vars[i]->val;
3380                 struct hist_field *save_var = hist_data->max_vars[i]->var;
3381                 u64 val;
3382
3383                 save_var_idx = save_var->var.idx;
3384
3385                 val = tracing_map_read_var(elt, save_var_idx);
3386
3387                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3388                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3389                                    (char *)(uintptr_t)(val));
3390                 } else
3391                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3392         }
3393 }
3394
3395 static void onmax_save(struct hist_trigger_data *hist_data,
3396                        struct tracing_map_elt *elt, void *rec,
3397                        struct ring_buffer_event *rbe,
3398                        struct action_data *data, u64 *var_ref_vals)
3399 {
3400         unsigned int max_idx = data->onmax.max_var->var.idx;
3401         unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx;
3402
3403         u64 var_val, max_val;
3404
3405         var_val = var_ref_vals[max_var_ref_idx];
3406         max_val = tracing_map_read_var(elt, max_idx);
3407
3408         if (var_val <= max_val)
3409                 return;
3410
3411         tracing_map_set_var(elt, max_idx, var_val);
3412
3413         update_max_vars(hist_data, elt, rbe, rec);
3414 }
3415
3416 static void onmax_destroy(struct action_data *data)
3417 {
3418         unsigned int i;
3419
3420         destroy_hist_field(data->onmax.max_var, 0);
3421         destroy_hist_field(data->onmax.var, 0);
3422
3423         kfree(data->onmax.var_str);
3424         kfree(data->onmax.fn_name);
3425
3426         for (i = 0; i < data->n_params; i++)
3427                 kfree(data->params[i]);
3428
3429         kfree(data);
3430 }
3431
3432 static int onmax_create(struct hist_trigger_data *hist_data,
3433                         struct action_data *data)
3434 {
3435         struct trace_event_file *file = hist_data->event_file;
3436         struct hist_field *var_field, *ref_field, *max_var;
3437         unsigned int var_ref_idx = hist_data->n_var_refs;
3438         struct field_var *field_var;
3439         char *onmax_var_str, *param;
3440         unsigned int i;
3441         int ret = 0;
3442
3443         onmax_var_str = data->onmax.var_str;
3444         if (onmax_var_str[0] != '$') {
3445                 hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str);
3446                 return -EINVAL;
3447         }
3448         onmax_var_str++;
3449
3450         var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str);
3451         if (!var_field) {
3452                 hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str);
3453                 return -EINVAL;
3454         }
3455
3456         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3457         if (!ref_field)
3458                 return -ENOMEM;
3459
3460         data->onmax.var = ref_field;
3461
3462         data->fn = onmax_save;
3463         data->onmax.max_var_ref_idx = var_ref_idx;
3464         max_var = create_var(hist_data, file, "max", sizeof(u64), "u64");
3465         if (IS_ERR(max_var)) {
3466                 hist_err("onmax: Couldn't create onmax variable: ", "max");
3467                 ret = PTR_ERR(max_var);
3468                 goto out;
3469         }
3470         data->onmax.max_var = max_var;
3471
3472         for (i = 0; i < data->n_params; i++) {
3473                 param = kstrdup(data->params[i], GFP_KERNEL);
3474                 if (!param) {
3475                         ret = -ENOMEM;
3476                         goto out;
3477                 }
3478
3479                 field_var = create_target_field_var(hist_data, NULL, NULL, param);
3480                 if (IS_ERR(field_var)) {
3481                         hist_err("onmax: Couldn't create field variable: ", param);
3482                         ret = PTR_ERR(field_var);
3483                         kfree(param);
3484                         goto out;
3485                 }
3486
3487                 hist_data->max_vars[hist_data->n_max_vars++] = field_var;
3488                 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3489                         hist_data->n_max_var_str++;
3490
3491                 kfree(param);
3492         }
3493  out:
3494         return ret;
3495 }
3496
3497 static int parse_action_params(char *params, struct action_data *data)
3498 {
3499         char *param, *saved_param;
3500         int ret = 0;
3501
3502         while (params) {
3503                 if (data->n_params >= SYNTH_FIELDS_MAX)
3504                         goto out;
3505
3506                 param = strsep(&params, ",");
3507                 if (!param) {
3508                         ret = -EINVAL;
3509                         goto out;
3510                 }
3511
3512                 param = strstrip(param);
3513                 if (strlen(param) < 2) {
3514                         hist_err("Invalid action param: ", param);
3515                         ret = -EINVAL;
3516                         goto out;
3517                 }
3518
3519                 saved_param = kstrdup(param, GFP_KERNEL);
3520                 if (!saved_param) {
3521                         ret = -ENOMEM;
3522                         goto out;
3523                 }
3524
3525                 data->params[data->n_params++] = saved_param;
3526         }
3527  out:
3528         return ret;
3529 }
3530
3531 static struct action_data *onmax_parse(char *str)
3532 {
3533         char *onmax_fn_name, *onmax_var_str;
3534         struct action_data *data;
3535         int ret = -EINVAL;
3536
3537         data = kzalloc(sizeof(*data), GFP_KERNEL);
3538         if (!data)
3539                 return ERR_PTR(-ENOMEM);
3540
3541         onmax_var_str = strsep(&str, ")");
3542         if (!onmax_var_str || !str) {
3543                 ret = -EINVAL;
3544                 goto free;
3545         }
3546
3547         data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL);
3548         if (!data->onmax.var_str) {
3549                 ret = -ENOMEM;
3550                 goto free;
3551         }
3552
3553         strsep(&str, ".");
3554         if (!str)
3555                 goto free;
3556
3557         onmax_fn_name = strsep(&str, "(");
3558         if (!onmax_fn_name || !str)
3559                 goto free;
3560
3561         if (str_has_prefix(onmax_fn_name, "save")) {
3562                 char *params = strsep(&str, ")");
3563
3564                 if (!params) {
3565                         ret = -EINVAL;
3566                         goto free;
3567                 }
3568
3569                 ret = parse_action_params(params, data);
3570                 if (ret)
3571                         goto free;
3572         } else
3573                 goto free;
3574
3575         data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL);
3576         if (!data->onmax.fn_name) {
3577                 ret = -ENOMEM;
3578                 goto free;
3579         }
3580  out:
3581         return data;
3582  free:
3583         onmax_destroy(data);
3584         data = ERR_PTR(ret);
3585         goto out;
3586 }
3587
3588 static void onmatch_destroy(struct action_data *data)
3589 {
3590         unsigned int i;
3591
3592         lockdep_assert_held(&event_mutex);
3593
3594         kfree(data->onmatch.match_event);
3595         kfree(data->onmatch.match_event_system);
3596         kfree(data->onmatch.synth_event_name);
3597
3598         for (i = 0; i < data->n_params; i++)
3599                 kfree(data->params[i]);
3600
3601         if (data->onmatch.synth_event)
3602                 data->onmatch.synth_event->ref--;
3603
3604         kfree(data);
3605 }
3606
3607 static void destroy_field_var(struct field_var *field_var)
3608 {
3609         if (!field_var)
3610                 return;
3611
3612         destroy_hist_field(field_var->var, 0);
3613         destroy_hist_field(field_var->val, 0);
3614
3615         kfree(field_var);
3616 }
3617
3618 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3619 {
3620         unsigned int i;
3621
3622         for (i = 0; i < hist_data->n_field_vars; i++)
3623                 destroy_field_var(hist_data->field_vars[i]);
3624 }
3625
3626 static void save_field_var(struct hist_trigger_data *hist_data,
3627                            struct field_var *field_var)
3628 {
3629         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3630
3631         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3632                 hist_data->n_field_var_str++;
3633 }
3634
3635
3636 static int check_synth_field(struct synth_event *event,
3637                              struct hist_field *hist_field,
3638                              unsigned int field_pos)
3639 {
3640         struct synth_field *field;
3641
3642         if (field_pos >= event->n_fields)
3643                 return -EINVAL;
3644
3645         field = event->fields[field_pos];
3646
3647         if (strcmp(field->type, hist_field->type) != 0)
3648                 return -EINVAL;
3649
3650         return 0;
3651 }
3652
3653 static struct hist_field *
3654 onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data,
3655                  char *system, char *event, char *var)
3656 {
3657         struct hist_field *hist_field;
3658
3659         var++; /* skip '$' */
3660
3661         hist_field = find_target_event_var(hist_data, system, event, var);
3662         if (!hist_field) {
3663                 if (!system) {
3664                         system = data->onmatch.match_event_system;
3665                         event = data->onmatch.match_event;
3666                 }
3667
3668                 hist_field = find_event_var(hist_data, system, event, var);
3669         }
3670
3671         if (!hist_field)
3672                 hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var);
3673
3674         return hist_field;
3675 }
3676
3677 static struct hist_field *
3678 onmatch_create_field_var(struct hist_trigger_data *hist_data,
3679                          struct action_data *data, char *system,
3680                          char *event, char *var)
3681 {
3682         struct hist_field *hist_field = NULL;
3683         struct field_var *field_var;
3684
3685         /*
3686          * First try to create a field var on the target event (the
3687          * currently being defined).  This will create a variable for
3688          * unqualified fields on the target event, or if qualified,
3689          * target fields that have qualified names matching the target.
3690          */
3691         field_var = create_target_field_var(hist_data, system, event, var);
3692
3693         if (field_var && !IS_ERR(field_var)) {
3694                 save_field_var(hist_data, field_var);
3695                 hist_field = field_var->var;
3696         } else {
3697                 field_var = NULL;
3698                 /*
3699                  * If no explicit system.event is specfied, default to
3700                  * looking for fields on the onmatch(system.event.xxx)
3701                  * event.
3702                  */
3703                 if (!system) {
3704                         system = data->onmatch.match_event_system;
3705                         event = data->onmatch.match_event;
3706                 }
3707
3708                 /*
3709                  * At this point, we're looking at a field on another
3710                  * event.  Because we can't modify a hist trigger on
3711                  * another event to add a variable for a field, we need
3712                  * to create a new trigger on that event and create the
3713                  * variable at the same time.
3714                  */
3715                 hist_field = create_field_var_hist(hist_data, system, event, var);
3716                 if (IS_ERR(hist_field))
3717                         goto free;
3718         }
3719  out:
3720         return hist_field;
3721  free:
3722         destroy_field_var(field_var);
3723         hist_field = NULL;
3724         goto out;
3725 }
3726
3727 static int onmatch_create(struct hist_trigger_data *hist_data,
3728                           struct trace_event_file *file,
3729                           struct action_data *data)
3730 {
3731         char *event_name, *param, *system = NULL;
3732         struct hist_field *hist_field, *var_ref;
3733         unsigned int i, var_ref_idx;
3734         unsigned int field_pos = 0;
3735         struct synth_event *event;
3736         int ret = 0;
3737
3738         lockdep_assert_held(&event_mutex);
3739
3740         event = find_synth_event(data->onmatch.synth_event_name);
3741         if (!event) {
3742                 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name);
3743                 return -EINVAL;
3744         }
3745         event->ref++;
3746
3747         var_ref_idx = hist_data->n_var_refs;
3748
3749         for (i = 0; i < data->n_params; i++) {
3750                 char *p;
3751
3752                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3753                 if (!param) {
3754                         ret = -ENOMEM;
3755                         goto err;
3756                 }
3757
3758                 system = strsep(&param, ".");
3759                 if (!param) {
3760                         param = (char *)system;
3761                         system = event_name = NULL;
3762                 } else {
3763                         event_name = strsep(&param, ".");
3764                         if (!param) {
3765                                 kfree(p);
3766                                 ret = -EINVAL;
3767                                 goto err;
3768                         }
3769                 }
3770
3771                 if (param[0] == '$')
3772                         hist_field = onmatch_find_var(hist_data, data, system,
3773                                                       event_name, param);
3774                 else
3775                         hist_field = onmatch_create_field_var(hist_data, data,
3776                                                               system,
3777                                                               event_name,
3778                                                               param);
3779
3780                 if (!hist_field) {
3781                         kfree(p);
3782                         ret = -EINVAL;
3783                         goto err;
3784                 }
3785
3786                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3787                         var_ref = create_var_ref(hist_data, hist_field,
3788                                                  system, event_name);
3789                         if (!var_ref) {
3790                                 kfree(p);
3791                                 ret = -ENOMEM;
3792                                 goto err;
3793                         }
3794
3795                         field_pos++;
3796                         kfree(p);
3797                         continue;
3798                 }
3799
3800                 hist_err_event("onmatch: Param type doesn't match synthetic event field type: ",
3801                                system, event_name, param);
3802                 kfree(p);
3803                 ret = -EINVAL;
3804                 goto err;
3805         }
3806
3807         if (field_pos != event->n_fields) {
3808                 hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name);
3809                 ret = -EINVAL;
3810                 goto err;
3811         }
3812
3813         data->fn = action_trace;
3814         data->onmatch.synth_event = event;
3815         data->onmatch.var_ref_idx = var_ref_idx;
3816  out:
3817         return ret;
3818  err:
3819         event->ref--;
3820
3821         goto out;
3822 }
3823
3824 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3825 {
3826         char *match_event, *match_event_system;
3827         char *synth_event_name, *params;
3828         struct action_data *data;
3829         int ret = -EINVAL;
3830
3831         data = kzalloc(sizeof(*data), GFP_KERNEL);
3832         if (!data)
3833                 return ERR_PTR(-ENOMEM);
3834
3835         match_event = strsep(&str, ")");
3836         if (!match_event || !str) {
3837                 hist_err("onmatch: Missing closing paren: ", match_event);
3838                 goto free;
3839         }
3840
3841         match_event_system = strsep(&match_event, ".");
3842         if (!match_event) {
3843                 hist_err("onmatch: Missing subsystem for match event: ", match_event_system);
3844                 goto free;
3845         }
3846
3847         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3848                 hist_err_event("onmatch: Invalid subsystem or event name: ",
3849                                match_event_system, match_event, NULL);
3850                 goto free;
3851         }
3852
3853         data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL);
3854         if (!data->onmatch.match_event) {
3855                 ret = -ENOMEM;
3856                 goto free;
3857         }
3858
3859         data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL);
3860         if (!data->onmatch.match_event_system) {
3861                 ret = -ENOMEM;
3862                 goto free;
3863         }
3864
3865         strsep(&str, ".");
3866         if (!str) {
3867                 hist_err("onmatch: Missing . after onmatch(): ", str);
3868                 goto free;
3869         }
3870
3871         synth_event_name = strsep(&str, "(");
3872         if (!synth_event_name || !str) {
3873                 hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name);
3874                 goto free;
3875         }
3876
3877         data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL);
3878         if (!data->onmatch.synth_event_name) {
3879                 ret = -ENOMEM;
3880                 goto free;
3881         }
3882
3883         params = strsep(&str, ")");
3884         if (!params || !str || (str && strlen(str))) {
3885                 hist_err("onmatch: Missing closing paramlist paren: ", params);
3886                 goto free;
3887         }
3888
3889         ret = parse_action_params(params, data);
3890         if (ret)
3891                 goto free;
3892  out:
3893         return data;
3894  free:
3895         onmatch_destroy(data);
3896         data = ERR_PTR(ret);
3897         goto out;
3898 }
3899
3900 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3901 {
3902         hist_data->fields[HITCOUNT_IDX] =
3903                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3904         if (!hist_data->fields[HITCOUNT_IDX])
3905                 return -ENOMEM;
3906
3907         hist_data->n_vals++;
3908         hist_data->n_fields++;
3909
3910         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3911                 return -EINVAL;
3912
3913         return 0;
3914 }
3915
3916 static int __create_val_field(struct hist_trigger_data *hist_data,
3917                               unsigned int val_idx,
3918                               struct trace_event_file *file,
3919                               char *var_name, char *field_str,
3920                               unsigned long flags)
3921 {
3922         struct hist_field *hist_field;
3923         int ret = 0;
3924
3925         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3926         if (IS_ERR(hist_field)) {
3927                 ret = PTR_ERR(hist_field);
3928                 goto out;
3929         }
3930
3931         hist_data->fields[val_idx] = hist_field;
3932
3933         ++hist_data->n_vals;
3934         ++hist_data->n_fields;
3935
3936         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3937                 ret = -EINVAL;
3938  out:
3939         return ret;
3940 }
3941
3942 static int create_val_field(struct hist_trigger_data *hist_data,
3943                             unsigned int val_idx,
3944                             struct trace_event_file *file,
3945                             char *field_str)
3946 {
3947         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3948                 return -EINVAL;
3949
3950         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3951 }
3952
3953 static int create_var_field(struct hist_trigger_data *hist_data,
3954                             unsigned int val_idx,
3955                             struct trace_event_file *file,
3956                             char *var_name, char *expr_str)
3957 {
3958         unsigned long flags = 0;
3959
3960         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3961                 return -EINVAL;
3962
3963         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3964                 hist_err("Variable already defined: ", var_name);
3965                 return -EINVAL;
3966         }
3967
3968         flags |= HIST_FIELD_FL_VAR;
3969         hist_data->n_vars++;
3970         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3971                 return -EINVAL;
3972
3973         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3974 }
3975
3976 static int create_val_fields(struct hist_trigger_data *hist_data,
3977                              struct trace_event_file *file)
3978 {
3979         char *fields_str, *field_str;
3980         unsigned int i, j = 1;
3981         int ret;
3982
3983         ret = create_hitcount_val(hist_data);
3984         if (ret)
3985                 goto out;
3986
3987         fields_str = hist_data->attrs->vals_str;
3988         if (!fields_str)
3989                 goto out;
3990
3991         strsep(&fields_str, "=");
3992         if (!fields_str)
3993                 goto out;
3994
3995         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3996                      j < TRACING_MAP_VALS_MAX; i++) {
3997                 field_str = strsep(&fields_str, ",");
3998                 if (!field_str)
3999                         break;
4000
4001                 if (strcmp(field_str, "hitcount") == 0)
4002                         continue;
4003
4004                 ret = create_val_field(hist_data, j++, file, field_str);
4005                 if (ret)
4006                         goto out;
4007         }
4008
4009         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4010                 ret = -EINVAL;
4011  out:
4012         return ret;
4013 }
4014
4015 static int create_key_field(struct hist_trigger_data *hist_data,
4016                             unsigned int key_idx,
4017                             unsigned int key_offset,
4018                             struct trace_event_file *file,
4019                             char *field_str)
4020 {
4021         struct hist_field *hist_field = NULL;
4022
4023         unsigned long flags = 0;
4024         unsigned int key_size;
4025         int ret = 0;
4026
4027         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4028                 return -EINVAL;
4029
4030         flags |= HIST_FIELD_FL_KEY;
4031
4032         if (strcmp(field_str, "stacktrace") == 0) {
4033                 flags |= HIST_FIELD_FL_STACKTRACE;
4034                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4035                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4036         } else {
4037                 hist_field = parse_expr(hist_data, file, field_str, flags,
4038                                         NULL, 0);
4039                 if (IS_ERR(hist_field)) {
4040                         ret = PTR_ERR(hist_field);
4041                         goto out;
4042                 }
4043
4044                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
4045                         hist_err("Using variable references as keys not supported: ", field_str);
4046                         destroy_hist_field(hist_field, 0);
4047                         ret = -EINVAL;
4048                         goto out;
4049                 }
4050
4051                 key_size = hist_field->size;
4052         }
4053
4054         hist_data->fields[key_idx] = hist_field;
4055
4056         key_size = ALIGN(key_size, sizeof(u64));
4057         hist_data->fields[key_idx]->size = key_size;
4058         hist_data->fields[key_idx]->offset = key_offset;
4059
4060         hist_data->key_size += key_size;
4061
4062         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4063                 ret = -EINVAL;
4064                 goto out;
4065         }
4066
4067         hist_data->n_keys++;
4068         hist_data->n_fields++;
4069
4070         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4071                 return -EINVAL;
4072
4073         ret = key_size;
4074  out:
4075         return ret;
4076 }
4077
4078 static int create_key_fields(struct hist_trigger_data *hist_data,
4079                              struct trace_event_file *file)
4080 {
4081         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4082         char *fields_str, *field_str;
4083         int ret = -EINVAL;
4084
4085         fields_str = hist_data->attrs->keys_str;
4086         if (!fields_str)
4087                 goto out;
4088
4089         strsep(&fields_str, "=");
4090         if (!fields_str)
4091                 goto out;
4092
4093         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4094                 field_str = strsep(&fields_str, ",");
4095                 if (!field_str)
4096                         break;
4097                 ret = create_key_field(hist_data, i, key_offset,
4098                                        file, field_str);
4099                 if (ret < 0)
4100                         goto out;
4101                 key_offset += ret;
4102         }
4103         if (fields_str) {
4104                 ret = -EINVAL;
4105                 goto out;
4106         }
4107         ret = 0;
4108  out:
4109         return ret;
4110 }
4111
4112 static int create_var_fields(struct hist_trigger_data *hist_data,
4113                              struct trace_event_file *file)
4114 {
4115         unsigned int i, j = hist_data->n_vals;
4116         int ret = 0;
4117
4118         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4119
4120         for (i = 0; i < n_vars; i++) {
4121                 char *var_name = hist_data->attrs->var_defs.name[i];
4122                 char *expr = hist_data->attrs->var_defs.expr[i];
4123
4124                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4125                 if (ret)
4126                         goto out;
4127         }
4128  out:
4129         return ret;
4130 }
4131
4132 static void free_var_defs(struct hist_trigger_data *hist_data)
4133 {
4134         unsigned int i;
4135
4136         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4137                 kfree(hist_data->attrs->var_defs.name[i]);
4138                 kfree(hist_data->attrs->var_defs.expr[i]);
4139         }
4140
4141         hist_data->attrs->var_defs.n_vars = 0;
4142 }
4143
4144 static int parse_var_defs(struct hist_trigger_data *hist_data)
4145 {
4146         char *s, *str, *var_name, *field_str;
4147         unsigned int i, j, n_vars = 0;
4148         int ret = 0;
4149
4150         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4151                 str = hist_data->attrs->assignment_str[i];
4152                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4153                         field_str = strsep(&str, ",");
4154                         if (!field_str)
4155                                 break;
4156
4157                         var_name = strsep(&field_str, "=");
4158                         if (!var_name || !field_str) {
4159                                 hist_err("Malformed assignment: ", var_name);
4160                                 ret = -EINVAL;
4161                                 goto free;
4162                         }
4163
4164                         if (n_vars == TRACING_MAP_VARS_MAX) {
4165                                 hist_err("Too many variables defined: ", var_name);
4166                                 ret = -EINVAL;
4167                                 goto free;
4168                         }
4169
4170                         s = kstrdup(var_name, GFP_KERNEL);
4171                         if (!s) {
4172                                 ret = -ENOMEM;
4173                                 goto free;
4174                         }
4175                         hist_data->attrs->var_defs.name[n_vars] = s;
4176
4177                         s = kstrdup(field_str, GFP_KERNEL);
4178                         if (!s) {
4179                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4180                                 ret = -ENOMEM;
4181                                 goto free;
4182                         }
4183                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4184
4185                         hist_data->attrs->var_defs.n_vars = n_vars;
4186                 }
4187         }
4188
4189         return ret;
4190  free:
4191         free_var_defs(hist_data);
4192
4193         return ret;
4194 }
4195
4196 static int create_hist_fields(struct hist_trigger_data *hist_data,
4197                               struct trace_event_file *file)
4198 {
4199         int ret;
4200
4201         ret = parse_var_defs(hist_data);
4202         if (ret)
4203                 goto out;
4204
4205         ret = create_val_fields(hist_data, file);
4206         if (ret)
4207                 goto out;
4208
4209         ret = create_var_fields(hist_data, file);
4210         if (ret)
4211                 goto out;
4212
4213         ret = create_key_fields(hist_data, file);
4214         if (ret)
4215                 goto out;
4216  out:
4217         free_var_defs(hist_data);
4218
4219         return ret;
4220 }
4221
4222 static int is_descending(const char *str)
4223 {
4224         if (!str)
4225                 return 0;
4226
4227         if (strcmp(str, "descending") == 0)
4228                 return 1;
4229
4230         if (strcmp(str, "ascending") == 0)
4231                 return 0;
4232
4233         return -EINVAL;
4234 }
4235
4236 static int create_sort_keys(struct hist_trigger_data *hist_data)
4237 {
4238         char *fields_str = hist_data->attrs->sort_key_str;
4239         struct tracing_map_sort_key *sort_key;
4240         int descending, ret = 0;
4241         unsigned int i, j, k;
4242
4243         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4244
4245         if (!fields_str)
4246                 goto out;
4247
4248         strsep(&fields_str, "=");
4249         if (!fields_str) {
4250                 ret = -EINVAL;
4251                 goto out;
4252         }
4253
4254         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4255                 struct hist_field *hist_field;
4256                 char *field_str, *field_name;
4257                 const char *test_name;
4258
4259                 sort_key = &hist_data->sort_keys[i];
4260
4261                 field_str = strsep(&fields_str, ",");
4262                 if (!field_str) {
4263                         if (i == 0)
4264                                 ret = -EINVAL;
4265                         break;
4266                 }
4267
4268                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4269                         ret = -EINVAL;
4270                         break;
4271                 }
4272
4273                 field_name = strsep(&field_str, ".");
4274                 if (!field_name) {
4275                         ret = -EINVAL;
4276                         break;
4277                 }
4278
4279                 if (strcmp(field_name, "hitcount") == 0) {
4280                         descending = is_descending(field_str);
4281                         if (descending < 0) {
4282                                 ret = descending;
4283                                 break;
4284                         }
4285                         sort_key->descending = descending;
4286                         continue;
4287                 }
4288
4289                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4290                         unsigned int idx;
4291
4292                         hist_field = hist_data->fields[j];
4293                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4294                                 continue;
4295
4296                         idx = k++;
4297
4298                         test_name = hist_field_name(hist_field, 0);
4299
4300                         if (strcmp(field_name, test_name) == 0) {
4301                                 sort_key->field_idx = idx;
4302                                 descending = is_descending(field_str);
4303                                 if (descending < 0) {
4304                                         ret = descending;
4305                                         goto out;
4306                                 }
4307                                 sort_key->descending = descending;
4308                                 break;
4309                         }
4310                 }
4311                 if (j == hist_data->n_fields) {
4312                         ret = -EINVAL;
4313                         break;
4314                 }
4315         }
4316
4317         hist_data->n_sort_keys = i;
4318  out:
4319         return ret;
4320 }
4321
4322 static void destroy_actions(struct hist_trigger_data *hist_data)
4323 {
4324         unsigned int i;
4325
4326         for (i = 0; i < hist_data->n_actions; i++) {
4327                 struct action_data *data = hist_data->actions[i];
4328
4329                 if (data->fn == action_trace)
4330                         onmatch_destroy(data);
4331                 else if (data->fn == onmax_save)
4332                         onmax_destroy(data);
4333                 else
4334                         kfree(data);
4335         }
4336 }
4337
4338 static int parse_actions(struct hist_trigger_data *hist_data)
4339 {
4340         struct trace_array *tr = hist_data->event_file->tr;
4341         struct action_data *data;
4342         unsigned int i;
4343         int ret = 0;
4344         char *str;
4345
4346         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4347                 str = hist_data->attrs->action_str[i];
4348
4349                 if (str_has_prefix(str, "onmatch(")) {
4350                         char *action_str = str + sizeof("onmatch(") - 1;
4351
4352                         data = onmatch_parse(tr, action_str);
4353                         if (IS_ERR(data)) {
4354                                 ret = PTR_ERR(data);
4355                                 break;
4356                         }
4357                         data->fn = action_trace;
4358                 } else if (str_has_prefix(str, "onmax(")) {
4359                         char *action_str = str + sizeof("onmax(") - 1;
4360
4361                         data = onmax_parse(action_str);
4362                         if (IS_ERR(data)) {
4363                                 ret = PTR_ERR(data);
4364                                 break;
4365                         }
4366                         data->fn = onmax_save;
4367                 } else {
4368                         ret = -EINVAL;
4369                         break;
4370                 }
4371
4372                 hist_data->actions[hist_data->n_actions++] = data;
4373         }
4374
4375         return ret;
4376 }
4377
4378 static int create_actions(struct hist_trigger_data *hist_data,
4379                           struct trace_event_file *file)
4380 {
4381         struct action_data *data;
4382         unsigned int i;
4383         int ret = 0;
4384
4385         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4386                 data = hist_data->actions[i];
4387
4388                 if (data->fn == action_trace) {
4389                         ret = onmatch_create(hist_data, file, data);
4390                         if (ret)
4391                                 return ret;
4392                 } else if (data->fn == onmax_save) {
4393                         ret = onmax_create(hist_data, data);
4394                         if (ret)
4395                                 return ret;
4396                 }
4397         }
4398
4399         return ret;
4400 }
4401
4402 static void print_actions(struct seq_file *m,
4403                           struct hist_trigger_data *hist_data,
4404                           struct tracing_map_elt *elt)
4405 {
4406         unsigned int i;
4407
4408         for (i = 0; i < hist_data->n_actions; i++) {
4409                 struct action_data *data = hist_data->actions[i];
4410
4411                 if (data->fn == onmax_save)
4412                         onmax_print(m, hist_data, elt, data);
4413         }
4414 }
4415
4416 static void print_onmax_spec(struct seq_file *m,
4417                              struct hist_trigger_data *hist_data,
4418                              struct action_data *data)
4419 {
4420         unsigned int i;
4421
4422         seq_puts(m, ":onmax(");
4423         seq_printf(m, "%s", data->onmax.var_str);
4424         seq_printf(m, ").%s(", data->onmax.fn_name);
4425
4426         for (i = 0; i < hist_data->n_max_vars; i++) {
4427                 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name);
4428                 if (i < hist_data->n_max_vars - 1)
4429                         seq_puts(m, ",");
4430         }
4431         seq_puts(m, ")");
4432 }
4433
4434 static void print_onmatch_spec(struct seq_file *m,
4435                                struct hist_trigger_data *hist_data,
4436                                struct action_data *data)
4437 {
4438         unsigned int i;
4439
4440         seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system,
4441                    data->onmatch.match_event);
4442
4443         seq_printf(m, "%s(", data->onmatch.synth_event->name);
4444
4445         for (i = 0; i < data->n_params; i++) {
4446                 if (i)
4447                         seq_puts(m, ",");
4448                 seq_printf(m, "%s", data->params[i]);
4449         }
4450
4451         seq_puts(m, ")");
4452 }
4453
4454 static bool actions_match(struct hist_trigger_data *hist_data,
4455                           struct hist_trigger_data *hist_data_test)
4456 {
4457         unsigned int i, j;
4458
4459         if (hist_data->n_actions != hist_data_test->n_actions)
4460                 return false;
4461
4462         for (i = 0; i < hist_data->n_actions; i++) {
4463                 struct action_data *data = hist_data->actions[i];
4464                 struct action_data *data_test = hist_data_test->actions[i];
4465
4466                 if (data->fn != data_test->fn)
4467                         return false;
4468
4469                 if (data->n_params != data_test->n_params)
4470                         return false;
4471
4472                 for (j = 0; j < data->n_params; j++) {
4473                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4474                                 return false;
4475                 }
4476
4477                 if (data->fn == action_trace) {
4478                         if (strcmp(data->onmatch.synth_event_name,
4479                                    data_test->onmatch.synth_event_name) != 0)
4480                                 return false;
4481                         if (strcmp(data->onmatch.match_event_system,
4482                                    data_test->onmatch.match_event_system) != 0)
4483                                 return false;
4484                         if (strcmp(data->onmatch.match_event,
4485                                    data_test->onmatch.match_event) != 0)
4486                                 return false;
4487                 } else if (data->fn == onmax_save) {
4488                         if (strcmp(data->onmax.var_str,
4489                                    data_test->onmax.var_str) != 0)
4490                                 return false;
4491                         if (strcmp(data->onmax.fn_name,
4492                                    data_test->onmax.fn_name) != 0)
4493                                 return false;
4494                 }
4495         }
4496
4497         return true;
4498 }
4499
4500
4501 static void print_actions_spec(struct seq_file *m,
4502                                struct hist_trigger_data *hist_data)
4503 {
4504         unsigned int i;
4505
4506         for (i = 0; i < hist_data->n_actions; i++) {
4507                 struct action_data *data = hist_data->actions[i];
4508
4509                 if (data->fn == action_trace)
4510                         print_onmatch_spec(m, hist_data, data);
4511                 else if (data->fn == onmax_save)
4512                         print_onmax_spec(m, hist_data, data);
4513         }
4514 }
4515
4516 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4517 {
4518         unsigned int i;
4519
4520         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4521                 kfree(hist_data->field_var_hists[i]->cmd);
4522                 kfree(hist_data->field_var_hists[i]);
4523         }
4524 }
4525
4526 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4527 {
4528         if (!hist_data)
4529                 return;
4530
4531         destroy_hist_trigger_attrs(hist_data->attrs);
4532         destroy_hist_fields(hist_data);
4533         tracing_map_destroy(hist_data->map);
4534
4535         destroy_actions(hist_data);
4536         destroy_field_vars(hist_data);
4537         destroy_field_var_hists(hist_data);
4538
4539         kfree(hist_data);
4540 }
4541
4542 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4543 {
4544         struct tracing_map *map = hist_data->map;
4545         struct ftrace_event_field *field;
4546         struct hist_field *hist_field;
4547         int i, idx = 0;
4548
4549         for_each_hist_field(i, hist_data) {
4550                 hist_field = hist_data->fields[i];
4551                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4552                         tracing_map_cmp_fn_t cmp_fn;
4553
4554                         field = hist_field->field;
4555
4556                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4557                                 cmp_fn = tracing_map_cmp_none;
4558                         else if (!field)
4559                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4560                                                              hist_field->is_signed);
4561                         else if (is_string_field(field))
4562                                 cmp_fn = tracing_map_cmp_string;
4563                         else
4564                                 cmp_fn = tracing_map_cmp_num(field->size,
4565                                                              field->is_signed);
4566                         idx = tracing_map_add_key_field(map,
4567                                                         hist_field->offset,
4568                                                         cmp_fn);
4569                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4570                         idx = tracing_map_add_sum_field(map);
4571
4572                 if (idx < 0)
4573                         return idx;
4574
4575                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4576                         idx = tracing_map_add_var(map);
4577                         if (idx < 0)
4578                                 return idx;
4579                         hist_field->var.idx = idx;
4580                         hist_field->var.hist_data = hist_data;
4581                 }
4582         }
4583
4584         return 0;
4585 }
4586
4587 static struct hist_trigger_data *
4588 create_hist_data(unsigned int map_bits,
4589                  struct hist_trigger_attrs *attrs,
4590                  struct trace_event_file *file,
4591                  bool remove)
4592 {
4593         const struct tracing_map_ops *map_ops = NULL;
4594         struct hist_trigger_data *hist_data;
4595         int ret = 0;
4596
4597         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4598         if (!hist_data)
4599                 return ERR_PTR(-ENOMEM);
4600
4601         hist_data->attrs = attrs;
4602         hist_data->remove = remove;
4603         hist_data->event_file = file;
4604
4605         ret = parse_actions(hist_data);
4606         if (ret)
4607                 goto free;
4608
4609         ret = create_hist_fields(hist_data, file);
4610         if (ret)
4611                 goto free;
4612
4613         ret = create_sort_keys(hist_data);
4614         if (ret)
4615                 goto free;
4616
4617         map_ops = &hist_trigger_elt_data_ops;
4618
4619         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4620                                             map_ops, hist_data);
4621         if (IS_ERR(hist_data->map)) {
4622                 ret = PTR_ERR(hist_data->map);
4623                 hist_data->map = NULL;
4624                 goto free;
4625         }
4626
4627         ret = create_tracing_map_fields(hist_data);
4628         if (ret)
4629                 goto free;
4630  out:
4631         return hist_data;
4632  free:
4633         hist_data->attrs = NULL;
4634
4635         destroy_hist_data(hist_data);
4636
4637         hist_data = ERR_PTR(ret);
4638
4639         goto out;
4640 }
4641
4642 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4643                                     struct tracing_map_elt *elt, void *rec,
4644                                     struct ring_buffer_event *rbe,
4645                                     u64 *var_ref_vals)
4646 {
4647         struct hist_elt_data *elt_data;
4648         struct hist_field *hist_field;
4649         unsigned int i, var_idx;
4650         u64 hist_val;
4651
4652         elt_data = elt->private_data;
4653         elt_data->var_ref_vals = var_ref_vals;
4654
4655         for_each_hist_val_field(i, hist_data) {
4656                 hist_field = hist_data->fields[i];
4657                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4658                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4659                         var_idx = hist_field->var.idx;
4660                         tracing_map_set_var(elt, var_idx, hist_val);
4661                         continue;
4662                 }
4663                 tracing_map_update_sum(elt, i, hist_val);
4664         }
4665
4666         for_each_hist_key_field(i, hist_data) {
4667                 hist_field = hist_data->fields[i];
4668                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4669                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4670                         var_idx = hist_field->var.idx;
4671                         tracing_map_set_var(elt, var_idx, hist_val);
4672                 }
4673         }
4674
4675         update_field_vars(hist_data, elt, rbe, rec);
4676 }
4677
4678 static inline void add_to_key(char *compound_key, void *key,
4679                               struct hist_field *key_field, void *rec)
4680 {
4681         size_t size = key_field->size;
4682
4683         if (key_field->flags & HIST_FIELD_FL_STRING) {
4684                 struct ftrace_event_field *field;
4685
4686                 field = key_field->field;
4687                 if (field->filter_type == FILTER_DYN_STRING)
4688                         size = *(u32 *)(rec + field->offset) >> 16;
4689                 else if (field->filter_type == FILTER_PTR_STRING)
4690                         size = strlen(key);
4691                 else if (field->filter_type == FILTER_STATIC_STRING)
4692                         size = field->size;
4693
4694                 /* ensure NULL-termination */
4695                 if (size > key_field->size - 1)
4696                         size = key_field->size - 1;
4697         }
4698
4699         memcpy(compound_key + key_field->offset, key, size);
4700 }
4701
4702 static void
4703 hist_trigger_actions(struct hist_trigger_data *hist_data,
4704                      struct tracing_map_elt *elt, void *rec,
4705                      struct ring_buffer_event *rbe, u64 *var_ref_vals)
4706 {
4707         struct action_data *data;
4708         unsigned int i;
4709
4710         for (i = 0; i < hist_data->n_actions; i++) {
4711                 data = hist_data->actions[i];
4712                 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals);
4713         }
4714 }
4715
4716 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4717                                struct ring_buffer_event *rbe)
4718 {
4719         struct hist_trigger_data *hist_data = data->private_data;
4720         bool use_compound_key = (hist_data->n_keys > 1);
4721         unsigned long entries[HIST_STACKTRACE_DEPTH];
4722         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4723         char compound_key[HIST_KEY_SIZE_MAX];
4724         struct tracing_map_elt *elt = NULL;
4725         struct stack_trace stacktrace;
4726         struct hist_field *key_field;
4727         u64 field_contents;
4728         void *key = NULL;
4729         unsigned int i;
4730
4731         memset(compound_key, 0, hist_data->key_size);
4732
4733         for_each_hist_key_field(i, hist_data) {
4734                 key_field = hist_data->fields[i];
4735
4736                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4737                         stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
4738                         stacktrace.entries = entries;
4739                         stacktrace.nr_entries = 0;
4740                         stacktrace.skip = HIST_STACKTRACE_SKIP;
4741
4742                         memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
4743                         save_stack_trace(&stacktrace);
4744
4745                         key = entries;
4746                 } else {
4747                         field_contents = key_field->fn(key_field, elt, rbe, rec);
4748                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4749                                 key = (void *)(unsigned long)field_contents;
4750                                 use_compound_key = true;
4751                         } else
4752                                 key = (void *)&field_contents;
4753                 }
4754
4755                 if (use_compound_key)
4756                         add_to_key(compound_key, key, key_field, rec);
4757         }
4758
4759         if (use_compound_key)
4760                 key = compound_key;
4761
4762         if (hist_data->n_var_refs &&
4763             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4764                 return;
4765
4766         elt = tracing_map_insert(hist_data->map, key);
4767         if (!elt)
4768                 return;
4769
4770         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4771
4772         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4773                 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals);
4774 }
4775
4776 static void hist_trigger_stacktrace_print(struct seq_file *m,
4777                                           unsigned long *stacktrace_entries,
4778                                           unsigned int max_entries)
4779 {
4780         char str[KSYM_SYMBOL_LEN];
4781         unsigned int spaces = 8;
4782         unsigned int i;
4783
4784         for (i = 0; i < max_entries; i++) {
4785                 if (stacktrace_entries[i] == ULONG_MAX)
4786                         return;
4787
4788                 seq_printf(m, "%*c", 1 + spaces, ' ');
4789                 sprint_symbol(str, stacktrace_entries[i]);
4790                 seq_printf(m, "%s\n", str);
4791         }
4792 }
4793
4794 static void
4795 hist_trigger_entry_print(struct seq_file *m,
4796                          struct hist_trigger_data *hist_data, void *key,
4797                          struct tracing_map_elt *elt)
4798 {
4799         struct hist_field *key_field;
4800         char str[KSYM_SYMBOL_LEN];
4801         bool multiline = false;
4802         const char *field_name;
4803         unsigned int i;
4804         u64 uval;
4805
4806         seq_puts(m, "{ ");
4807
4808         for_each_hist_key_field(i, hist_data) {
4809                 key_field = hist_data->fields[i];
4810
4811                 if (i > hist_data->n_vals)
4812                         seq_puts(m, ", ");
4813
4814                 field_name = hist_field_name(key_field, 0);
4815
4816                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4817                         uval = *(u64 *)(key + key_field->offset);
4818                         seq_printf(m, "%s: %llx", field_name, uval);
4819                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4820                         uval = *(u64 *)(key + key_field->offset);
4821                         sprint_symbol_no_offset(str, uval);
4822                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4823                                    uval, str);
4824                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4825                         uval = *(u64 *)(key + key_field->offset);
4826                         sprint_symbol(str, uval);
4827                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4828                                    uval, str);
4829                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4830                         struct hist_elt_data *elt_data = elt->private_data;
4831                         char *comm;
4832
4833                         if (WARN_ON_ONCE(!elt_data))
4834                                 return;
4835
4836                         comm = elt_data->comm;
4837
4838                         uval = *(u64 *)(key + key_field->offset);
4839                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4840                                    comm, uval);
4841                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4842                         const char *syscall_name;
4843
4844                         uval = *(u64 *)(key + key_field->offset);
4845                         syscall_name = get_syscall_name(uval);
4846                         if (!syscall_name)
4847                                 syscall_name = "unknown_syscall";
4848
4849                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4850                                    syscall_name, uval);
4851                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4852                         seq_puts(m, "stacktrace:\n");
4853                         hist_trigger_stacktrace_print(m,
4854                                                       key + key_field->offset,
4855                                                       HIST_STACKTRACE_DEPTH);
4856                         multiline = true;
4857                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4858                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4859                                    *(u64 *)(key + key_field->offset));
4860                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4861                         seq_printf(m, "%s: %-50s", field_name,
4862                                    (char *)(key + key_field->offset));
4863                 } else {
4864                         uval = *(u64 *)(key + key_field->offset);
4865                         seq_printf(m, "%s: %10llu", field_name, uval);
4866                 }
4867         }
4868
4869         if (!multiline)
4870                 seq_puts(m, " ");
4871
4872         seq_puts(m, "}");
4873
4874         seq_printf(m, " hitcount: %10llu",
4875                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4876
4877         for (i = 1; i < hist_data->n_vals; i++) {
4878                 field_name = hist_field_name(hist_data->fields[i], 0);
4879
4880                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4881                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4882                         continue;
4883
4884                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4885                         seq_printf(m, "  %s: %10llx", field_name,
4886                                    tracing_map_read_sum(elt, i));
4887                 } else {
4888                         seq_printf(m, "  %s: %10llu", field_name,
4889                                    tracing_map_read_sum(elt, i));
4890                 }
4891         }
4892
4893         print_actions(m, hist_data, elt);
4894
4895         seq_puts(m, "\n");
4896 }
4897
4898 static int print_entries(struct seq_file *m,
4899                          struct hist_trigger_data *hist_data)
4900 {
4901         struct tracing_map_sort_entry **sort_entries = NULL;
4902         struct tracing_map *map = hist_data->map;
4903         int i, n_entries;
4904
4905         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4906                                              hist_data->n_sort_keys,
4907                                              &sort_entries);
4908         if (n_entries < 0)
4909                 return n_entries;
4910
4911         for (i = 0; i < n_entries; i++)
4912                 hist_trigger_entry_print(m, hist_data,
4913                                          sort_entries[i]->key,
4914                                          sort_entries[i]->elt);
4915
4916         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4917
4918         return n_entries;
4919 }
4920
4921 static void hist_trigger_show(struct seq_file *m,
4922                               struct event_trigger_data *data, int n)
4923 {
4924         struct hist_trigger_data *hist_data;
4925         int n_entries;
4926
4927         if (n > 0)
4928                 seq_puts(m, "\n\n");
4929
4930         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4931         data->ops->print(m, data->ops, data);
4932         seq_puts(m, "#\n\n");
4933
4934         hist_data = data->private_data;
4935         n_entries = print_entries(m, hist_data);
4936         if (n_entries < 0)
4937                 n_entries = 0;
4938
4939         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4940                    (u64)atomic64_read(&hist_data->map->hits),
4941                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4942 }
4943
4944 static int hist_show(struct seq_file *m, void *v)
4945 {
4946         struct event_trigger_data *data;
4947         struct trace_event_file *event_file;
4948         int n = 0, ret = 0;
4949
4950         mutex_lock(&event_mutex);
4951
4952         event_file = event_file_data(m->private);
4953         if (unlikely(!event_file)) {
4954                 ret = -ENODEV;
4955                 goto out_unlock;
4956         }
4957
4958         list_for_each_entry_rcu(data, &event_file->triggers, list) {
4959                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4960                         hist_trigger_show(m, data, n++);
4961         }
4962
4963         if (have_hist_err()) {
4964                 seq_printf(m, "\nERROR: %s\n", hist_err_str);
4965                 seq_printf(m, "  Last command: %s\n", last_hist_cmd);
4966         }
4967
4968  out_unlock:
4969         mutex_unlock(&event_mutex);
4970
4971         return ret;
4972 }
4973
4974 static int event_hist_open(struct inode *inode, struct file *file)
4975 {
4976         return single_open(file, hist_show, file);
4977 }
4978
4979 const struct file_operations event_hist_fops = {
4980         .open = event_hist_open,
4981         .read = seq_read,
4982         .llseek = seq_lseek,
4983         .release = single_release,
4984 };
4985
4986 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
4987 {
4988         const char *field_name = hist_field_name(hist_field, 0);
4989
4990         if (hist_field->var.name)
4991                 seq_printf(m, "%s=", hist_field->var.name);
4992
4993         if (hist_field->flags & HIST_FIELD_FL_CPU)
4994                 seq_puts(m, "cpu");
4995         else if (field_name) {
4996                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
4997                     hist_field->flags & HIST_FIELD_FL_ALIAS)
4998                         seq_putc(m, '$');
4999                 seq_printf(m, "%s", field_name);
5000         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5001                 seq_puts(m, "common_timestamp");
5002
5003         if (hist_field->flags) {
5004                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5005                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5006                         const char *flags = get_hist_field_flags(hist_field);
5007
5008                         if (flags)
5009                                 seq_printf(m, ".%s", flags);
5010                 }
5011         }
5012 }
5013
5014 static int event_hist_trigger_print(struct seq_file *m,
5015                                     struct event_trigger_ops *ops,
5016                                     struct event_trigger_data *data)
5017 {
5018         struct hist_trigger_data *hist_data = data->private_data;
5019         struct hist_field *field;
5020         bool have_var = false;
5021         unsigned int i;
5022
5023         seq_puts(m, "hist:");
5024
5025         if (data->name)
5026                 seq_printf(m, "%s:", data->name);
5027
5028         seq_puts(m, "keys=");
5029
5030         for_each_hist_key_field(i, hist_data) {
5031                 field = hist_data->fields[i];
5032
5033                 if (i > hist_data->n_vals)
5034                         seq_puts(m, ",");
5035
5036                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5037                         seq_puts(m, "stacktrace");
5038                 else
5039                         hist_field_print(m, field);
5040         }
5041
5042         seq_puts(m, ":vals=");
5043
5044         for_each_hist_val_field(i, hist_data) {
5045                 field = hist_data->fields[i];
5046                 if (field->flags & HIST_FIELD_FL_VAR) {
5047                         have_var = true;
5048                         continue;
5049                 }
5050
5051                 if (i == HITCOUNT_IDX)
5052                         seq_puts(m, "hitcount");
5053                 else {
5054                         seq_puts(m, ",");
5055                         hist_field_print(m, field);
5056                 }
5057         }
5058
5059         if (have_var) {
5060                 unsigned int n = 0;
5061
5062                 seq_puts(m, ":");
5063
5064                 for_each_hist_val_field(i, hist_data) {
5065                         field = hist_data->fields[i];
5066
5067                         if (field->flags & HIST_FIELD_FL_VAR) {
5068                                 if (n++)
5069                                         seq_puts(m, ",");
5070                                 hist_field_print(m, field);
5071                         }
5072                 }
5073         }
5074
5075         seq_puts(m, ":sort=");
5076
5077         for (i = 0; i < hist_data->n_sort_keys; i++) {
5078                 struct tracing_map_sort_key *sort_key;
5079                 unsigned int idx, first_key_idx;
5080
5081                 /* skip VAR vals */
5082                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5083
5084                 sort_key = &hist_data->sort_keys[i];
5085                 idx = sort_key->field_idx;
5086
5087                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5088                         return -EINVAL;
5089
5090                 if (i > 0)
5091                         seq_puts(m, ",");
5092
5093                 if (idx == HITCOUNT_IDX)
5094                         seq_puts(m, "hitcount");
5095                 else {
5096                         if (idx >= first_key_idx)
5097                                 idx += hist_data->n_vars;
5098                         hist_field_print(m, hist_data->fields[idx]);
5099                 }
5100
5101                 if (sort_key->descending)
5102                         seq_puts(m, ".descending");
5103         }
5104         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5105         if (hist_data->enable_timestamps)
5106                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5107
5108         print_actions_spec(m, hist_data);
5109
5110         if (data->filter_str)
5111                 seq_printf(m, " if %s", data->filter_str);
5112
5113         if (data->paused)
5114                 seq_puts(m, " [paused]");
5115         else
5116                 seq_puts(m, " [active]");
5117
5118         seq_putc(m, '\n');
5119
5120         return 0;
5121 }
5122
5123 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5124                                    struct event_trigger_data *data)
5125 {
5126         struct hist_trigger_data *hist_data = data->private_data;
5127
5128         if (!data->ref && hist_data->attrs->name)
5129                 save_named_trigger(hist_data->attrs->name, data);
5130
5131         data->ref++;
5132
5133         return 0;
5134 }
5135
5136 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5137 {
5138         struct trace_event_file *file;
5139         unsigned int i;
5140         char *cmd;
5141         int ret;
5142
5143         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5144                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5145                 cmd = hist_data->field_var_hists[i]->cmd;
5146                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5147                                               "!hist", "hist", cmd);
5148         }
5149 }
5150
5151 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5152                                     struct event_trigger_data *data)
5153 {
5154         struct hist_trigger_data *hist_data = data->private_data;
5155
5156         if (WARN_ON_ONCE(data->ref <= 0))
5157                 return;
5158
5159         data->ref--;
5160         if (!data->ref) {
5161                 if (data->name)
5162                         del_named_trigger(data);
5163
5164                 trigger_data_free(data);
5165
5166                 remove_hist_vars(hist_data);
5167
5168                 unregister_field_var_hists(hist_data);
5169
5170                 destroy_hist_data(hist_data);
5171         }
5172 }
5173
5174 static struct event_trigger_ops event_hist_trigger_ops = {
5175         .func                   = event_hist_trigger,
5176         .print                  = event_hist_trigger_print,
5177         .init                   = event_hist_trigger_init,
5178         .free                   = event_hist_trigger_free,
5179 };
5180
5181 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5182                                          struct event_trigger_data *data)
5183 {
5184         data->ref++;
5185
5186         save_named_trigger(data->named_data->name, data);
5187
5188         event_hist_trigger_init(ops, data->named_data);
5189
5190         return 0;
5191 }
5192
5193 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5194                                           struct event_trigger_data *data)
5195 {
5196         if (WARN_ON_ONCE(data->ref <= 0))
5197                 return;
5198
5199         event_hist_trigger_free(ops, data->named_data);
5200
5201         data->ref--;
5202         if (!data->ref) {
5203                 del_named_trigger(data);
5204                 trigger_data_free(data);
5205         }
5206 }
5207
5208 static struct event_trigger_ops event_hist_trigger_named_ops = {
5209         .func                   = event_hist_trigger,
5210         .print                  = event_hist_trigger_print,
5211         .init                   = event_hist_trigger_named_init,
5212         .free                   = event_hist_trigger_named_free,
5213 };
5214
5215 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5216                                                             char *param)
5217 {
5218         return &event_hist_trigger_ops;
5219 }
5220
5221 static void hist_clear(struct event_trigger_data *data)
5222 {
5223         struct hist_trigger_data *hist_data = data->private_data;
5224
5225         if (data->name)
5226                 pause_named_trigger(data);
5227
5228         tracepoint_synchronize_unregister();
5229
5230         tracing_map_clear(hist_data->map);
5231
5232         if (data->name)
5233                 unpause_named_trigger(data);
5234 }
5235
5236 static bool compatible_field(struct ftrace_event_field *field,
5237                              struct ftrace_event_field *test_field)
5238 {
5239         if (field == test_field)
5240                 return true;
5241         if (field == NULL || test_field == NULL)
5242                 return false;
5243         if (strcmp(field->name, test_field->name) != 0)
5244                 return false;
5245         if (strcmp(field->type, test_field->type) != 0)
5246                 return false;
5247         if (field->size != test_field->size)
5248                 return false;
5249         if (field->is_signed != test_field->is_signed)
5250                 return false;
5251
5252         return true;
5253 }
5254
5255 static bool hist_trigger_match(struct event_trigger_data *data,
5256                                struct event_trigger_data *data_test,
5257                                struct event_trigger_data *named_data,
5258                                bool ignore_filter)
5259 {
5260         struct tracing_map_sort_key *sort_key, *sort_key_test;
5261         struct hist_trigger_data *hist_data, *hist_data_test;
5262         struct hist_field *key_field, *key_field_test;
5263         unsigned int i;
5264
5265         if (named_data && (named_data != data_test) &&
5266             (named_data != data_test->named_data))
5267                 return false;
5268
5269         if (!named_data && is_named_trigger(data_test))
5270                 return false;
5271
5272         hist_data = data->private_data;
5273         hist_data_test = data_test->private_data;
5274
5275         if (hist_data->n_vals != hist_data_test->n_vals ||
5276             hist_data->n_fields != hist_data_test->n_fields ||
5277             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5278                 return false;
5279
5280         if (!ignore_filter) {
5281                 if ((data->filter_str && !data_test->filter_str) ||
5282                    (!data->filter_str && data_test->filter_str))
5283                         return false;
5284         }
5285
5286         for_each_hist_field(i, hist_data) {
5287                 key_field = hist_data->fields[i];
5288                 key_field_test = hist_data_test->fields[i];
5289
5290                 if (key_field->flags != key_field_test->flags)
5291                         return false;
5292                 if (!compatible_field(key_field->field, key_field_test->field))
5293                         return false;
5294                 if (key_field->offset != key_field_test->offset)
5295                         return false;
5296                 if (key_field->size != key_field_test->size)
5297                         return false;
5298                 if (key_field->is_signed != key_field_test->is_signed)
5299                         return false;
5300                 if (!!key_field->var.name != !!key_field_test->var.name)
5301                         return false;
5302                 if (key_field->var.name &&
5303                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5304                         return false;
5305         }
5306
5307         for (i = 0; i < hist_data->n_sort_keys; i++) {
5308                 sort_key = &hist_data->sort_keys[i];
5309                 sort_key_test = &hist_data_test->sort_keys[i];
5310
5311                 if (sort_key->field_idx != sort_key_test->field_idx ||
5312                     sort_key->descending != sort_key_test->descending)
5313                         return false;
5314         }
5315
5316         if (!ignore_filter && data->filter_str &&
5317             (strcmp(data->filter_str, data_test->filter_str) != 0))
5318                 return false;
5319
5320         if (!actions_match(hist_data, hist_data_test))
5321                 return false;
5322
5323         return true;
5324 }
5325
5326 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5327                                  struct event_trigger_data *data,
5328                                  struct trace_event_file *file)
5329 {
5330         struct hist_trigger_data *hist_data = data->private_data;
5331         struct event_trigger_data *test, *named_data = NULL;
5332         int ret = 0;
5333
5334         if (hist_data->attrs->name) {
5335                 named_data = find_named_trigger(hist_data->attrs->name);
5336                 if (named_data) {
5337                         if (!hist_trigger_match(data, named_data, named_data,
5338                                                 true)) {
5339                                 hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name);
5340                                 ret = -EINVAL;
5341                                 goto out;
5342                         }
5343                 }
5344         }
5345
5346         if (hist_data->attrs->name && !named_data)
5347                 goto new;
5348
5349         list_for_each_entry_rcu(test, &file->triggers, list) {
5350                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5351                         if (!hist_trigger_match(data, test, named_data, false))
5352                                 continue;
5353                         if (hist_data->attrs->pause)
5354                                 test->paused = true;
5355                         else if (hist_data->attrs->cont)
5356                                 test->paused = false;
5357                         else if (hist_data->attrs->clear)
5358                                 hist_clear(test);
5359                         else {
5360                                 hist_err("Hist trigger already exists", NULL);
5361                                 ret = -EEXIST;
5362                         }
5363                         goto out;
5364                 }
5365         }
5366  new:
5367         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5368                 hist_err("Can't clear or continue a nonexistent hist trigger", NULL);
5369                 ret = -ENOENT;
5370                 goto out;
5371         }
5372
5373         if (hist_data->attrs->pause)
5374                 data->paused = true;
5375
5376         if (named_data) {
5377                 data->private_data = named_data->private_data;
5378                 set_named_trigger_data(data, named_data);
5379                 data->ops = &event_hist_trigger_named_ops;
5380         }
5381
5382         if (data->ops->init) {
5383                 ret = data->ops->init(data->ops, data);
5384                 if (ret < 0)
5385                         goto out;
5386         }
5387
5388         if (hist_data->enable_timestamps) {
5389                 char *clock = hist_data->attrs->clock;
5390
5391                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5392                 if (ret) {
5393                         hist_err("Couldn't set trace_clock: ", clock);
5394                         goto out;
5395                 }
5396
5397                 tracing_set_time_stamp_abs(file->tr, true);
5398         }
5399
5400         if (named_data)
5401                 destroy_hist_data(hist_data);
5402
5403         ret++;
5404  out:
5405         return ret;
5406 }
5407
5408 static int hist_trigger_enable(struct event_trigger_data *data,
5409                                struct trace_event_file *file)
5410 {
5411         int ret = 0;
5412
5413         list_add_tail_rcu(&data->list, &file->triggers);
5414
5415         update_cond_flag(file);
5416
5417         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5418                 list_del_rcu(&data->list);
5419                 update_cond_flag(file);
5420                 ret--;
5421         }
5422
5423         return ret;
5424 }
5425
5426 static bool have_hist_trigger_match(struct event_trigger_data *data,
5427                                     struct trace_event_file *file)
5428 {
5429         struct hist_trigger_data *hist_data = data->private_data;
5430         struct event_trigger_data *test, *named_data = NULL;
5431         bool match = false;
5432
5433         if (hist_data->attrs->name)
5434                 named_data = find_named_trigger(hist_data->attrs->name);
5435
5436         list_for_each_entry_rcu(test, &file->triggers, list) {
5437                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5438                         if (hist_trigger_match(data, test, named_data, false)) {
5439                                 match = true;
5440                                 break;
5441                         }
5442                 }
5443         }
5444
5445         return match;
5446 }
5447
5448 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5449                                     struct trace_event_file *file)
5450 {
5451         struct hist_trigger_data *hist_data = data->private_data;
5452         struct event_trigger_data *test, *named_data = NULL;
5453
5454         if (hist_data->attrs->name)
5455                 named_data = find_named_trigger(hist_data->attrs->name);
5456
5457         list_for_each_entry_rcu(test, &file->triggers, list) {
5458                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5459                         if (!hist_trigger_match(data, test, named_data, false))
5460                                 continue;
5461                         hist_data = test->private_data;
5462                         if (check_var_refs(hist_data))
5463                                 return true;
5464                         break;
5465                 }
5466         }
5467
5468         return false;
5469 }
5470
5471 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5472                                     struct event_trigger_data *data,
5473                                     struct trace_event_file *file)
5474 {
5475         struct hist_trigger_data *hist_data = data->private_data;
5476         struct event_trigger_data *test, *named_data = NULL;
5477         bool unregistered = false;
5478
5479         if (hist_data->attrs->name)
5480                 named_data = find_named_trigger(hist_data->attrs->name);
5481
5482         list_for_each_entry_rcu(test, &file->triggers, list) {
5483                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5484                         if (!hist_trigger_match(data, test, named_data, false))
5485                                 continue;
5486                         unregistered = true;
5487                         list_del_rcu(&test->list);
5488                         trace_event_trigger_enable_disable(file, 0);
5489                         update_cond_flag(file);
5490                         break;
5491                 }
5492         }
5493
5494         if (unregistered && test->ops->free)
5495                 test->ops->free(test->ops, test);
5496
5497         if (hist_data->enable_timestamps) {
5498                 if (!hist_data->remove || unregistered)
5499                         tracing_set_time_stamp_abs(file->tr, false);
5500         }
5501 }
5502
5503 static bool hist_file_check_refs(struct trace_event_file *file)
5504 {
5505         struct hist_trigger_data *hist_data;
5506         struct event_trigger_data *test;
5507
5508         list_for_each_entry_rcu(test, &file->triggers, list) {
5509                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5510                         hist_data = test->private_data;
5511                         if (check_var_refs(hist_data))
5512                                 return true;
5513                 }
5514         }
5515
5516         return false;
5517 }
5518
5519 static void hist_unreg_all(struct trace_event_file *file)
5520 {
5521         struct event_trigger_data *test, *n;
5522         struct hist_trigger_data *hist_data;
5523         struct synth_event *se;
5524         const char *se_name;
5525
5526         lockdep_assert_held(&event_mutex);
5527
5528         if (hist_file_check_refs(file))
5529                 return;
5530
5531         list_for_each_entry_safe(test, n, &file->triggers, list) {
5532                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5533                         hist_data = test->private_data;
5534                         list_del_rcu(&test->list);
5535                         trace_event_trigger_enable_disable(file, 0);
5536
5537                         se_name = trace_event_name(file->event_call);
5538                         se = find_synth_event(se_name);
5539                         if (se)
5540                                 se->ref--;
5541
5542                         update_cond_flag(file);
5543                         if (hist_data->enable_timestamps)
5544                                 tracing_set_time_stamp_abs(file->tr, false);
5545                         if (test->ops->free)
5546                                 test->ops->free(test->ops, test);
5547                 }
5548         }
5549 }
5550
5551 static int event_hist_trigger_func(struct event_command *cmd_ops,
5552                                    struct trace_event_file *file,
5553                                    char *glob, char *cmd, char *param)
5554 {
5555         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5556         struct event_trigger_data *trigger_data;
5557         struct hist_trigger_attrs *attrs;
5558         struct event_trigger_ops *trigger_ops;
5559         struct hist_trigger_data *hist_data;
5560         struct synth_event *se;
5561         const char *se_name;
5562         bool remove = false;
5563         char *trigger, *p;
5564         int ret = 0;
5565
5566         lockdep_assert_held(&event_mutex);
5567
5568         if (glob && strlen(glob)) {
5569                 last_cmd_set(param);
5570                 hist_err_clear();
5571         }
5572
5573         if (!param)
5574                 return -EINVAL;
5575
5576         if (glob[0] == '!')
5577                 remove = true;
5578
5579         /*
5580          * separate the trigger from the filter (k:v [if filter])
5581          * allowing for whitespace in the trigger
5582          */
5583         p = trigger = param;
5584         do {
5585                 p = strstr(p, "if");
5586                 if (!p)
5587                         break;
5588                 if (p == param)
5589                         return -EINVAL;
5590                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5591                         p++;
5592                         continue;
5593                 }
5594                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5595                         return -EINVAL;
5596                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5597                         p++;
5598                         continue;
5599                 }
5600                 break;
5601         } while (p);
5602
5603         if (!p)
5604                 param = NULL;
5605         else {
5606                 *(p - 1) = '\0';
5607                 param = strstrip(p);
5608                 trigger = strstrip(trigger);
5609         }
5610
5611         attrs = parse_hist_trigger_attrs(trigger);
5612         if (IS_ERR(attrs))
5613                 return PTR_ERR(attrs);
5614
5615         if (attrs->map_bits)
5616                 hist_trigger_bits = attrs->map_bits;
5617
5618         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5619         if (IS_ERR(hist_data)) {
5620                 destroy_hist_trigger_attrs(attrs);
5621                 return PTR_ERR(hist_data);
5622         }
5623
5624         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5625
5626         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5627         if (!trigger_data) {
5628                 ret = -ENOMEM;
5629                 goto out_free;
5630         }
5631
5632         trigger_data->count = -1;
5633         trigger_data->ops = trigger_ops;
5634         trigger_data->cmd_ops = cmd_ops;
5635
5636         INIT_LIST_HEAD(&trigger_data->list);
5637         RCU_INIT_POINTER(trigger_data->filter, NULL);
5638
5639         trigger_data->private_data = hist_data;
5640
5641         /* if param is non-empty, it's supposed to be a filter */
5642         if (param && cmd_ops->set_filter) {
5643                 ret = cmd_ops->set_filter(param, trigger_data, file);
5644                 if (ret < 0)
5645                         goto out_free;
5646         }
5647
5648         if (remove) {
5649                 if (!have_hist_trigger_match(trigger_data, file))
5650                         goto out_free;
5651
5652                 if (hist_trigger_check_refs(trigger_data, file)) {
5653                         ret = -EBUSY;
5654                         goto out_free;
5655                 }
5656
5657                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5658                 se_name = trace_event_name(file->event_call);
5659                 se = find_synth_event(se_name);
5660                 if (se)
5661                         se->ref--;
5662                 ret = 0;
5663                 goto out_free;
5664         }
5665
5666         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5667         /*
5668          * The above returns on success the # of triggers registered,
5669          * but if it didn't register any it returns zero.  Consider no
5670          * triggers registered a failure too.
5671          */
5672         if (!ret) {
5673                 if (!(attrs->pause || attrs->cont || attrs->clear))
5674                         ret = -ENOENT;
5675                 goto out_free;
5676         } else if (ret < 0)
5677                 goto out_free;
5678
5679         if (get_named_trigger_data(trigger_data))
5680                 goto enable;
5681
5682         if (has_hist_vars(hist_data))
5683                 save_hist_vars(hist_data);
5684
5685         ret = create_actions(hist_data, file);
5686         if (ret)
5687                 goto out_unreg;
5688
5689         ret = tracing_map_init(hist_data->map);
5690         if (ret)
5691                 goto out_unreg;
5692 enable:
5693         ret = hist_trigger_enable(trigger_data, file);
5694         if (ret)
5695                 goto out_unreg;
5696
5697         se_name = trace_event_name(file->event_call);
5698         se = find_synth_event(se_name);
5699         if (se)
5700                 se->ref++;
5701         /* Just return zero, not the number of registered triggers */
5702         ret = 0;
5703  out:
5704         if (ret == 0)
5705                 hist_err_clear();
5706
5707         return ret;
5708  out_unreg:
5709         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5710  out_free:
5711         if (cmd_ops->set_filter)
5712                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5713
5714         remove_hist_vars(hist_data);
5715
5716         kfree(trigger_data);
5717
5718         destroy_hist_data(hist_data);
5719         goto out;
5720 }
5721
5722 static struct event_command trigger_hist_cmd = {
5723         .name                   = "hist",
5724         .trigger_type           = ETT_EVENT_HIST,
5725         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5726         .func                   = event_hist_trigger_func,
5727         .reg                    = hist_register_trigger,
5728         .unreg                  = hist_unregister_trigger,
5729         .unreg_all              = hist_unreg_all,
5730         .get_trigger_ops        = event_hist_get_trigger_ops,
5731         .set_filter             = set_trigger_filter,
5732 };
5733
5734 __init int register_trigger_hist_cmd(void)
5735 {
5736         int ret;
5737
5738         ret = register_event_command(&trigger_hist_cmd);
5739         WARN_ON(ret < 0);
5740
5741         return ret;
5742 }
5743
5744 static void
5745 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5746                     struct ring_buffer_event *event)
5747 {
5748         struct enable_trigger_data *enable_data = data->private_data;
5749         struct event_trigger_data *test;
5750
5751         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
5752                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5753                         if (enable_data->enable)
5754                                 test->paused = false;
5755                         else
5756                                 test->paused = true;
5757                 }
5758         }
5759 }
5760
5761 static void
5762 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5763                           struct ring_buffer_event *event)
5764 {
5765         if (!data->count)
5766                 return;
5767
5768         if (data->count != -1)
5769                 (data->count)--;
5770
5771         hist_enable_trigger(data, rec, event);
5772 }
5773
5774 static struct event_trigger_ops hist_enable_trigger_ops = {
5775         .func                   = hist_enable_trigger,
5776         .print                  = event_enable_trigger_print,
5777         .init                   = event_trigger_init,
5778         .free                   = event_enable_trigger_free,
5779 };
5780
5781 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5782         .func                   = hist_enable_count_trigger,
5783         .print                  = event_enable_trigger_print,
5784         .init                   = event_trigger_init,
5785         .free                   = event_enable_trigger_free,
5786 };
5787
5788 static struct event_trigger_ops hist_disable_trigger_ops = {
5789         .func                   = hist_enable_trigger,
5790         .print                  = event_enable_trigger_print,
5791         .init                   = event_trigger_init,
5792         .free                   = event_enable_trigger_free,
5793 };
5794
5795 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5796         .func                   = hist_enable_count_trigger,
5797         .print                  = event_enable_trigger_print,
5798         .init                   = event_trigger_init,
5799         .free                   = event_enable_trigger_free,
5800 };
5801
5802 static struct event_trigger_ops *
5803 hist_enable_get_trigger_ops(char *cmd, char *param)
5804 {
5805         struct event_trigger_ops *ops;
5806         bool enable;
5807
5808         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5809
5810         if (enable)
5811                 ops = param ? &hist_enable_count_trigger_ops :
5812                         &hist_enable_trigger_ops;
5813         else
5814                 ops = param ? &hist_disable_count_trigger_ops :
5815                         &hist_disable_trigger_ops;
5816
5817         return ops;
5818 }
5819
5820 static void hist_enable_unreg_all(struct trace_event_file *file)
5821 {
5822         struct event_trigger_data *test, *n;
5823
5824         list_for_each_entry_safe(test, n, &file->triggers, list) {
5825                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5826                         list_del_rcu(&test->list);
5827                         update_cond_flag(file);
5828                         trace_event_trigger_enable_disable(file, 0);
5829                         if (test->ops->free)
5830                                 test->ops->free(test->ops, test);
5831                 }
5832         }
5833 }
5834
5835 static struct event_command trigger_hist_enable_cmd = {
5836         .name                   = ENABLE_HIST_STR,
5837         .trigger_type           = ETT_HIST_ENABLE,
5838         .func                   = event_enable_trigger_func,
5839         .reg                    = event_enable_register_trigger,
5840         .unreg                  = event_enable_unregister_trigger,
5841         .unreg_all              = hist_enable_unreg_all,
5842         .get_trigger_ops        = hist_enable_get_trigger_ops,
5843         .set_filter             = set_trigger_filter,
5844 };
5845
5846 static struct event_command trigger_hist_disable_cmd = {
5847         .name                   = DISABLE_HIST_STR,
5848         .trigger_type           = ETT_HIST_ENABLE,
5849         .func                   = event_enable_trigger_func,
5850         .reg                    = event_enable_register_trigger,
5851         .unreg                  = event_enable_unregister_trigger,
5852         .unreg_all              = hist_enable_unreg_all,
5853         .get_trigger_ops        = hist_enable_get_trigger_ops,
5854         .set_filter             = set_trigger_filter,
5855 };
5856
5857 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5858 {
5859         unregister_event_command(&trigger_hist_enable_cmd);
5860         unregister_event_command(&trigger_hist_disable_cmd);
5861 }
5862
5863 __init int register_trigger_hist_enable_disable_cmds(void)
5864 {
5865         int ret;
5866
5867         ret = register_event_command(&trigger_hist_enable_cmd);
5868         if (WARN_ON(ret < 0))
5869                 return ret;
5870         ret = register_event_command(&trigger_hist_disable_cmd);
5871         if (WARN_ON(ret < 0))
5872                 unregister_trigger_hist_enable_disable_cmds();
5873
5874         return ret;
5875 }
5876
5877 static __init int trace_events_hist_init(void)
5878 {
5879         struct dentry *entry = NULL;
5880         struct dentry *d_tracer;
5881         int err = 0;
5882
5883         err = dyn_event_register(&synth_event_ops);
5884         if (err) {
5885                 pr_warn("Could not register synth_event_ops\n");
5886                 return err;
5887         }
5888
5889         d_tracer = tracing_init_dentry();
5890         if (IS_ERR(d_tracer)) {
5891                 err = PTR_ERR(d_tracer);
5892                 goto err;
5893         }
5894
5895         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
5896                                     NULL, &synth_events_fops);
5897         if (!entry) {
5898                 err = -ENODEV;
5899                 goto err;
5900         }
5901
5902         return err;
5903  err:
5904         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
5905
5906         return err;
5907 }
5908
5909 fs_initcall(trace_events_hist_init);