kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace.h"
  23 #include "trace_dynevent.h"
  24
  25 #define SYNTH_SYSTEM            "synthetic"
  26 #define SYNTH_FIELDS_MAX        32
  27
  28 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  29
  30 #define ERRORS                                                          \
  31         C(NONE,                 "No error"),                            \
  32         C(DUPLICATE_VAR,        "Variable already defined"),            \
  33         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  34         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  35         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  36         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  37         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  38         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  39         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  40         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  41         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  42         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  43         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  44         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  45         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  46         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  47         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  48         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  49         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  50         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  51         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  52         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  53         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  54         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  55         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  56         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  57         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  58         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  59         C(INVALID_PARAM,        "Invalid action param"),                \
  60         C(ACTION_NOT_FOUND,     "No action found"),                     \
  61         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  62         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  63         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  64         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  65         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  66         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  67         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  68         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  69         C(FIELD_NOT_FOUND,      "Couldn't find field"),
  70
  71 #undef C
  72 #define C(a, b)         HIST_ERR_##a
  73
  74 enum { ERRORS };
  75
  76 #undef C
  77 #define C(a, b)         b
  78
  79 static const char *err_text[] = { ERRORS };
  80
  81 struct hist_field;
  82
  83 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  84                                 struct tracing_map_elt *elt,
  85                                 struct ring_buffer_event *rbe,
  86                                 void *event);
  87
  88 #define HIST_FIELD_OPERANDS_MAX 2
  89 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  90 #define HIST_ACTIONS_MAX        8
  91
  92 enum field_op_id {
  93         FIELD_OP_NONE,
  94         FIELD_OP_PLUS,
  95         FIELD_OP_MINUS,
  96         FIELD_OP_UNARY_MINUS,
  97 };
  98
  99 /*
 100  * A hist_var (histogram variable) contains variable information for
 101  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 102  * flag set.  A hist_var has a variable name e.g. ts0, and is
 103  * associated with a given histogram trigger, as specified by
 104  * hist_data.  The hist_var idx is the unique index assigned to the
 105  * variable by the hist trigger's tracing_map.  The idx is what is
 106  * used to set a variable's value and, by a variable reference, to
 107  * retrieve it.
 108  */
 109 struct hist_var {
 110         char                            *name;
 111         struct hist_trigger_data        *hist_data;
 112         unsigned int                    idx;
 113 };
 114
 115 struct hist_field {
 116         struct ftrace_event_field       *field;
 117         unsigned long                   flags;
 118         hist_field_fn_t                 fn;
 119         unsigned int                    size;
 120         unsigned int                    offset;
 121         unsigned int                    is_signed;
 122         const char                      *type;
 123         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 124         struct hist_trigger_data        *hist_data;
 125
 126         /*
 127          * Variable fields contain variable-specific info in var.
 128          */
 129         struct hist_var                 var;
 130         enum field_op_id                operator;
 131         char                            *system;
 132         char                            *event_name;
 133
 134         /*
 135          * The name field is used for EXPR and VAR_REF fields.  VAR
 136          * fields contain the variable name in var.name.
 137          */
 138         char                            *name;
 139
 140         /*
 141          * When a histogram trigger is hit, if it has any references
 142          * to variables, the values of those variables are collected
 143          * into a var_ref_vals array by resolve_var_refs().  The
 144          * current value of each variable is read from the tracing_map
 145          * using the hist field's hist_var.idx and entered into the
 146          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 147          */
 148         unsigned int                    var_ref_idx;
 149         bool                            read_once;
 150 };
 151
 152 static u64 hist_field_none(struct hist_field *field,
 153                            struct tracing_map_elt *elt,
 154                            struct ring_buffer_event *rbe,
 155                            void *event)
 156 {
 157         return 0;
 158 }
 159
 160 static u64 hist_field_counter(struct hist_field *field,
 161                               struct tracing_map_elt *elt,
 162                               struct ring_buffer_event *rbe,
 163                               void *event)
 164 {
 165         return 1;
 166 }
 167
 168 static u64 hist_field_string(struct hist_field *hist_field,
 169                              struct tracing_map_elt *elt,
 170                              struct ring_buffer_event *rbe,
 171                              void *event)
 172 {
 173         char *addr = (char *)(event + hist_field->field->offset);
 174
 175         return (u64)(unsigned long)addr;
 176 }
 177
 178 static u64 hist_field_dynstring(struct hist_field *hist_field,
 179                                 struct tracing_map_elt *elt,
 180                                 struct ring_buffer_event *rbe,
 181                                 void *event)
 182 {
 183         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 184         int str_loc = str_item & 0xffff;
 185         char *addr = (char *)(event + str_loc);
 186
 187         return (u64)(unsigned long)addr;
 188 }
 189
 190 static u64 hist_field_pstring(struct hist_field *hist_field,
 191                               struct tracing_map_elt *elt,
 192                               struct ring_buffer_event *rbe,
 193                               void *event)
 194 {
 195         char **addr = (char **)(event + hist_field->field->offset);
 196
 197         return (u64)(unsigned long)*addr;
 198 }
 199
 200 static u64 hist_field_log2(struct hist_field *hist_field,
 201                            struct tracing_map_elt *elt,
 202                            struct ring_buffer_event *rbe,
 203                            void *event)
 204 {
 205         struct hist_field *operand = hist_field->operands[0];
 206
 207         u64 val = operand->fn(operand, elt, rbe, event);
 208
 209         return (u64) ilog2(roundup_pow_of_two(val));
 210 }
 211
 212 static u64 hist_field_plus(struct hist_field *hist_field,
 213                            struct tracing_map_elt *elt,
 214                            struct ring_buffer_event *rbe,
 215                            void *event)
 216 {
 217         struct hist_field *operand1 = hist_field->operands[0];
 218         struct hist_field *operand2 = hist_field->operands[1];
 219
 220         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 221         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 222
 223         return val1 + val2;
 224 }
 225
 226 static u64 hist_field_minus(struct hist_field *hist_field,
 227                             struct tracing_map_elt *elt,
 228                             struct ring_buffer_event *rbe,
 229                             void *event)
 230 {
 231         struct hist_field *operand1 = hist_field->operands[0];
 232         struct hist_field *operand2 = hist_field->operands[1];
 233
 234         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 235         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 236
 237         return val1 - val2;
 238 }
 239
 240 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 241                                   struct tracing_map_elt *elt,
 242                                   struct ring_buffer_event *rbe,
 243                                   void *event)
 244 {
 245         struct hist_field *operand = hist_field->operands[0];
 246
 247         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 248         u64 val = (u64)-sval;
 249
 250         return val;
 251 }
 252
 253 #define DEFINE_HIST_FIELD_FN(type)                                      \
 254         static u64 hist_field_##type(struct hist_field *hist_field,     \
 255                                      struct tracing_map_elt *elt,       \
 256                                      struct ring_buffer_event *rbe,     \
 257                                      void *event)                       \
 258 {                                                                       \
 259         type *addr = (type *)(event + hist_field->field->offset);       \
 260                                                                         \
 261         return (u64)(unsigned long)*addr;                               \
 262 }
 263
 264 DEFINE_HIST_FIELD_FN(s64);
 265 DEFINE_HIST_FIELD_FN(u64);
 266 DEFINE_HIST_FIELD_FN(s32);
 267 DEFINE_HIST_FIELD_FN(u32);
 268 DEFINE_HIST_FIELD_FN(s16);
 269 DEFINE_HIST_FIELD_FN(u16);
 270 DEFINE_HIST_FIELD_FN(s8);
 271 DEFINE_HIST_FIELD_FN(u8);
 272
 273 #define for_each_hist_field(i, hist_data)       \
 274         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 275
 276 #define for_each_hist_val_field(i, hist_data)   \
 277         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 278
 279 #define for_each_hist_key_field(i, hist_data)   \
 280         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 281
 282 #define HIST_STACKTRACE_DEPTH   16
 283 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 284 #define HIST_STACKTRACE_SKIP    5
 285
 286 #define HITCOUNT_IDX            0
 287 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 288
 289 enum hist_field_flags {
 290         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 291         HIST_FIELD_FL_KEY               = 1 << 1,
 292         HIST_FIELD_FL_STRING            = 1 << 2,
 293         HIST_FIELD_FL_HEX               = 1 << 3,
 294         HIST_FIELD_FL_SYM               = 1 << 4,
 295         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 296         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 297         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 298         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 299         HIST_FIELD_FL_LOG2              = 1 << 9,
 300         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 301         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 302         HIST_FIELD_FL_VAR               = 1 << 12,
 303         HIST_FIELD_FL_EXPR              = 1 << 13,
 304         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 305         HIST_FIELD_FL_CPU               = 1 << 15,
 306         HIST_FIELD_FL_ALIAS             = 1 << 16,
 307 };
 308
 309 struct var_defs {
 310         unsigned int    n_vars;
 311         char            *name[TRACING_MAP_VARS_MAX];
 312         char            *expr[TRACING_MAP_VARS_MAX];
 313 };
 314
 315 struct hist_trigger_attrs {
 316         char            *keys_str;
 317         char            *vals_str;
 318         char            *sort_key_str;
 319         char            *name;
 320         char            *clock;
 321         bool            pause;
 322         bool            cont;
 323         bool            clear;
 324         bool            ts_in_usecs;
 325         unsigned int    map_bits;
 326
 327         char            *assignment_str[TRACING_MAP_VARS_MAX];
 328         unsigned int    n_assignments;
 329
 330         char            *action_str[HIST_ACTIONS_MAX];
 331         unsigned int    n_actions;
 332
 333         struct var_defs var_defs;
 334 };
 335
 336 struct field_var {
 337         struct hist_field       *var;
 338         struct hist_field       *val;
 339 };
 340
 341 struct field_var_hist {
 342         struct hist_trigger_data        *hist_data;
 343         char                            *cmd;
 344 };
 345
 346 struct hist_trigger_data {
 347         struct hist_field               *fields[HIST_FIELDS_MAX];
 348         unsigned int                    n_vals;
 349         unsigned int                    n_keys;
 350         unsigned int                    n_fields;
 351         unsigned int                    n_vars;
 352         unsigned int                    key_size;
 353         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 354         unsigned int                    n_sort_keys;
 355         struct trace_event_file         *event_file;
 356         struct hist_trigger_attrs       *attrs;
 357         struct tracing_map              *map;
 358         bool                            enable_timestamps;
 359         bool                            remove;
 360         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 361         unsigned int                    n_var_refs;
 362
 363         struct action_data              *actions[HIST_ACTIONS_MAX];
 364         unsigned int                    n_actions;
 365
 366         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 367         unsigned int                    n_field_vars;
 368         unsigned int                    n_field_var_str;
 369         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 370         unsigned int                    n_field_var_hists;
 371
 372         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 373         unsigned int                    n_save_vars;
 374         unsigned int                    n_save_var_str;
 375 };
 376
 377 static int synth_event_create(int argc, const char **argv);
 378 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 379 static int synth_event_release(struct dyn_event *ev);
 380 static bool synth_event_is_busy(struct dyn_event *ev);
 381 static bool synth_event_match(const char *system, const char *event,
 382                         int argc, const char **argv, struct dyn_event *ev);
 383
 384 static struct dyn_event_operations synth_event_ops = {
 385         .create = synth_event_create,
 386         .show = synth_event_show,
 387         .is_busy = synth_event_is_busy,
 388         .free = synth_event_release,
 389         .match = synth_event_match,
 390 };
 391
 392 struct synth_field {
 393         char *type;
 394         char *name;
 395         size_t size;
 396         bool is_signed;
 397         bool is_string;
 398 };
 399
 400 struct synth_event {
 401         struct dyn_event                        devent;
 402         int                                     ref;
 403         char                                    *name;
 404         struct synth_field                      **fields;
 405         unsigned int                            n_fields;
 406         unsigned int                            n_u64;
 407         struct trace_event_class                class;
 408         struct trace_event_call                 call;
 409         struct tracepoint                       *tp;
 410 };
 411
 412 static bool is_synth_event(struct dyn_event *ev)
 413 {
 414         return ev->ops == &synth_event_ops;
 415 }
 416
 417 static struct synth_event *to_synth_event(struct dyn_event *ev)
 418 {
 419         return container_of(ev, struct synth_event, devent);
 420 }
 421
 422 static bool synth_event_is_busy(struct dyn_event *ev)
 423 {
 424         struct synth_event *event = to_synth_event(ev);
 425
 426         return event->ref != 0;
 427 }
 428
 429 static bool synth_event_match(const char *system, const char *event,
 430                         int argc, const char **argv, struct dyn_event *ev)
 431 {
 432         struct synth_event *sev = to_synth_event(ev);
 433
 434         return strcmp(sev->name, event) == 0 &&
 435                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 436 }
 437
 438 struct action_data;
 439
 440 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 441                              struct tracing_map_elt *elt, void *rec,
 442                              struct ring_buffer_event *rbe, void *key,
 443                              struct action_data *data, u64 *var_ref_vals);
 444
 445 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 446
 447 enum handler_id {
 448         HANDLER_ONMATCH = 1,
 449         HANDLER_ONMAX,
 450         HANDLER_ONCHANGE,
 451 };
 452
 453 enum action_id {
 454         ACTION_SAVE = 1,
 455         ACTION_TRACE,
 456         ACTION_SNAPSHOT,
 457 };
 458
 459 struct action_data {
 460         enum handler_id         handler;
 461         enum action_id          action;
 462         char                    *action_name;
 463         action_fn_t             fn;
 464
 465         unsigned int            n_params;
 466         char                    *params[SYNTH_FIELDS_MAX];
 467
 468         /*
 469          * When a histogram trigger is hit, the values of any
 470          * references to variables, including variables being passed
 471          * as parameters to synthetic events, are collected into a
 472          * var_ref_vals array.  This var_ref_idx is the index of the
 473          * first param in the array to be passed to the synthetic
 474          * event invocation.
 475          */
 476         unsigned int            var_ref_idx;
 477         struct synth_event      *synth_event;
 478         bool                    use_trace_keyword;
 479         char                    *synth_event_name;
 480
 481         union {
 482                 struct {
 483                         char                    *event;
 484                         char                    *event_system;
 485                 } match_data;
 486
 487                 struct {
 488                         /*
 489                          * var_str contains the $-unstripped variable
 490                          * name referenced by var_ref, and used when
 491                          * printing the action.  Because var_ref
 492                          * creation is deferred to create_actions(),
 493                          * we need a per-action way to save it until
 494                          * then, thus var_str.
 495                          */
 496                         char                    *var_str;
 497
 498                         /*
 499                          * var_ref refers to the variable being
 500                          * tracked e.g onmax($var).
 501                          */
 502                         struct hist_field       *var_ref;
 503
 504                         /*
 505                          * track_var contains the 'invisible' tracking
 506                          * variable created to keep the current
 507                          * e.g. max value.
 508                          */
 509                         struct hist_field       *track_var;
 510
 511                         check_track_val_fn_t    check_val;
 512                         action_fn_t             save_data;
 513                 } track_data;
 514         };
 515 };
 516
 517 struct track_data {
 518         u64                             track_val;
 519         bool                            updated;
 520
 521         unsigned int                    key_len;
 522         void                            *key;
 523         struct tracing_map_elt          elt;
 524
 525         struct action_data              *action_data;
 526         struct hist_trigger_data        *hist_data;
 527 };
 528
 529 struct hist_elt_data {
 530         char *comm;
 531         u64 *var_ref_vals;
 532         char *field_var_str[SYNTH_FIELDS_MAX];
 533 };
 534
 535 struct snapshot_context {
 536         struct tracing_map_elt  *elt;
 537         void                    *key;
 538 };
 539
 540 static void track_data_free(struct track_data *track_data)
 541 {
 542         struct hist_elt_data *elt_data;
 543
 544         if (!track_data)
 545                 return;
 546
 547         kfree(track_data->key);
 548
 549         elt_data = track_data->elt.private_data;
 550         if (elt_data) {
 551                 kfree(elt_data->comm);
 552                 kfree(elt_data);
 553         }
 554
 555         kfree(track_data);
 556 }
 557
 558 static struct track_data *track_data_alloc(unsigned int key_len,
 559                                            struct action_data *action_data,
 560                                            struct hist_trigger_data *hist_data)
 561 {
 562         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 563         struct hist_elt_data *elt_data;
 564
 565         if (!data)
 566                 return ERR_PTR(-ENOMEM);
 567
 568         data->key = kzalloc(key_len, GFP_KERNEL);
 569         if (!data->key) {
 570                 track_data_free(data);
 571                 return ERR_PTR(-ENOMEM);
 572         }
 573
 574         data->key_len = key_len;
 575         data->action_data = action_data;
 576         data->hist_data = hist_data;
 577
 578         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 579         if (!elt_data) {
 580                 track_data_free(data);
 581                 return ERR_PTR(-ENOMEM);
 582         }
 583         data->elt.private_data = elt_data;
 584
 585         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 586         if (!elt_data->comm) {
 587                 track_data_free(data);
 588                 return ERR_PTR(-ENOMEM);
 589         }
 590
 591         return data;
 592 }
 593
 594 static char last_cmd[MAX_FILTER_STR_VAL];
 595 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 596
 597 static int errpos(char *str)
 598 {
 599         return err_pos(last_cmd, str);
 600 }
 601
 602 static void last_cmd_set(struct trace_event_file *file, char *str)
 603 {
 604         const char *system = NULL, *name = NULL;
 605         struct trace_event_call *call;
 606
 607         if (!str)
 608                 return;
 609
 610         strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1);
 611
 612         if (file) {
 613                 call = file->event_call;
 614
 615                 system = call->class->system;
 616                 if (system) {
 617                         name = trace_event_name(call);
 618                         if (!name)
 619                                 system = NULL;
 620                 }
 621         }
 622
 623         if (system)
 624                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 625 }
 626
 627 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 628 {
 629         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 630                         err_type, err_pos);
 631 }
 632
 633 static void hist_err_clear(void)
 634 {
 635         last_cmd[0] = '\0';
 636         last_cmd_loc[0] = '\0';
 637 }
 638
 639 struct synth_trace_event {
 640         struct trace_entry      ent;
 641         u64                     fields[];
 642 };
 643
 644 static int synth_event_define_fields(struct trace_event_call *call)
 645 {
 646         struct synth_trace_event trace;
 647         int offset = offsetof(typeof(trace), fields);
 648         struct synth_event *event = call->data;
 649         unsigned int i, size, n_u64;
 650         char *name, *type;
 651         bool is_signed;
 652         int ret = 0;
 653
 654         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 655                 size = event->fields[i]->size;
 656                 is_signed = event->fields[i]->is_signed;
 657                 type = event->fields[i]->type;
 658                 name = event->fields[i]->name;
 659                 ret = trace_define_field(call, type, name, offset, size,
 660                                          is_signed, FILTER_OTHER);
 661                 if (ret)
 662                         break;
 663
 664                 if (event->fields[i]->is_string) {
 665                         offset += STR_VAR_LEN_MAX;
 666                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 667                 } else {
 668                         offset += sizeof(u64);
 669                         n_u64++;
 670                 }
 671         }
 672
 673         event->n_u64 = n_u64;
 674
 675         return ret;
 676 }
 677
 678 static bool synth_field_signed(char *type)
 679 {
 680         if (str_has_prefix(type, "u"))
 681                 return false;
 682         if (strcmp(type, "gfp_t") == 0)
 683                 return false;
 684
 685         return true;
 686 }
 687
 688 static int synth_field_is_string(char *type)
 689 {
 690         if (strstr(type, "char[") != NULL)
 691                 return true;
 692
 693         return false;
 694 }
 695
 696 static int synth_field_string_size(char *type)
 697 {
 698         char buf[4], *end, *start;
 699         unsigned int len;
 700         int size, err;
 701
 702         start = strstr(type, "char[");
 703         if (start == NULL)
 704                 return -EINVAL;
 705         start += sizeof("char[") - 1;
 706
 707         end = strchr(type, ']');
 708         if (!end || end < start)
 709                 return -EINVAL;
 710
 711         len = end - start;
 712         if (len > 3)
 713                 return -EINVAL;
 714
 715         strncpy(buf, start, len);
 716         buf[len] = '\0';
 717
 718         err = kstrtouint(buf, 0, &size);
 719         if (err)
 720                 return err;
 721
 722         if (size > STR_VAR_LEN_MAX)
 723                 return -EINVAL;
 724
 725         return size;
 726 }
 727
 728 static int synth_field_size(char *type)
 729 {
 730         int size = 0;
 731
 732         if (strcmp(type, "s64") == 0)
 733                 size = sizeof(s64);
 734         else if (strcmp(type, "u64") == 0)
 735                 size = sizeof(u64);
 736         else if (strcmp(type, "s32") == 0)
 737                 size = sizeof(s32);
 738         else if (strcmp(type, "u32") == 0)
 739                 size = sizeof(u32);
 740         else if (strcmp(type, "s16") == 0)
 741                 size = sizeof(s16);
 742         else if (strcmp(type, "u16") == 0)
 743                 size = sizeof(u16);
 744         else if (strcmp(type, "s8") == 0)
 745                 size = sizeof(s8);
 746         else if (strcmp(type, "u8") == 0)
 747                 size = sizeof(u8);
 748         else if (strcmp(type, "char") == 0)
 749                 size = sizeof(char);
 750         else if (strcmp(type, "unsigned char") == 0)
 751                 size = sizeof(unsigned char);
 752         else if (strcmp(type, "int") == 0)
 753                 size = sizeof(int);
 754         else if (strcmp(type, "unsigned int") == 0)
 755                 size = sizeof(unsigned int);
 756         else if (strcmp(type, "long") == 0)
 757                 size = sizeof(long);
 758         else if (strcmp(type, "unsigned long") == 0)
 759                 size = sizeof(unsigned long);
 760         else if (strcmp(type, "pid_t") == 0)
 761                 size = sizeof(pid_t);
 762         else if (strcmp(type, "gfp_t") == 0)
 763                 size = sizeof(gfp_t);
 764         else if (synth_field_is_string(type))
 765                 size = synth_field_string_size(type);
 766
 767         return size;
 768 }
 769
 770 static const char *synth_field_fmt(char *type)
 771 {
 772         const char *fmt = "%llu";
 773
 774         if (strcmp(type, "s64") == 0)
 775                 fmt = "%lld";
 776         else if (strcmp(type, "u64") == 0)
 777                 fmt = "%llu";
 778         else if (strcmp(type, "s32") == 0)
 779                 fmt = "%d";
 780         else if (strcmp(type, "u32") == 0)
 781                 fmt = "%u";
 782         else if (strcmp(type, "s16") == 0)
 783                 fmt = "%d";
 784         else if (strcmp(type, "u16") == 0)
 785                 fmt = "%u";
 786         else if (strcmp(type, "s8") == 0)
 787                 fmt = "%d";
 788         else if (strcmp(type, "u8") == 0)
 789                 fmt = "%u";
 790         else if (strcmp(type, "char") == 0)
 791                 fmt = "%d";
 792         else if (strcmp(type, "unsigned char") == 0)
 793                 fmt = "%u";
 794         else if (strcmp(type, "int") == 0)
 795                 fmt = "%d";
 796         else if (strcmp(type, "unsigned int") == 0)
 797                 fmt = "%u";
 798         else if (strcmp(type, "long") == 0)
 799                 fmt = "%ld";
 800         else if (strcmp(type, "unsigned long") == 0)
 801                 fmt = "%lu";
 802         else if (strcmp(type, "pid_t") == 0)
 803                 fmt = "%d";
 804         else if (strcmp(type, "gfp_t") == 0)
 805                 fmt = "%x";
 806         else if (synth_field_is_string(type))
 807                 fmt = "%s";
 808
 809         return fmt;
 810 }
 811
 812 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 813                                            int flags,
 814                                            struct trace_event *event)
 815 {
 816         struct trace_array *tr = iter->tr;
 817         struct trace_seq *s = &iter->seq;
 818         struct synth_trace_event *entry;
 819         struct synth_event *se;
 820         unsigned int i, n_u64;
 821         char print_fmt[32];
 822         const char *fmt;
 823
 824         entry = (struct synth_trace_event *)iter->ent;
 825         se = container_of(event, struct synth_event, call.event);
 826
 827         trace_seq_printf(s, "%s: ", se->name);
 828
 829         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 830                 if (trace_seq_has_overflowed(s))
 831                         goto end;
 832
 833                 fmt = synth_field_fmt(se->fields[i]->type);
 834
 835                 /* parameter types */
 836                 if (tr && tr->trace_flags & TRACE_ITER_VERBOSE)
 837                         trace_seq_printf(s, "%s ", fmt);
 838
 839                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 840
 841                 /* parameter values */
 842                 if (se->fields[i]->is_string) {
 843                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 844                                          (char *)&entry->fields[n_u64],
 845                                          i == se->n_fields - 1 ? "" : " ");
 846                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 847                 } else {
 848                         struct trace_print_flags __flags[] = {
 849                             __def_gfpflag_names, {-1, NULL} };
 850
 851                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 852                                          entry->fields[n_u64],
 853                                          i == se->n_fields - 1 ? "" : " ");
 854
 855                         if (strcmp(se->fields[i]->type, "gfp_t") == 0) {
 856                                 trace_seq_puts(s, " (");
 857                                 trace_print_flags_seq(s, "|",
 858                                                       entry->fields[n_u64],
 859                                                       __flags);
 860                                 trace_seq_putc(s, ')');
 861                         }
 862                         n_u64++;
 863                 }
 864         }
 865 end:
 866         trace_seq_putc(s, '\n');
 867
 868         return trace_handle_return(s);
 869 }
 870
 871 static struct trace_event_functions synth_event_funcs = {
 872         .trace          = print_synth_event
 873 };
 874
 875 static notrace void trace_event_raw_event_synth(void *__data,
 876                                                 u64 *var_ref_vals,
 877                                                 unsigned int var_ref_idx)
 878 {
 879         struct trace_event_file *trace_file = __data;
 880         struct synth_trace_event *entry;
 881         struct trace_event_buffer fbuffer;
 882         struct trace_buffer *buffer;
 883         struct synth_event *event;
 884         unsigned int i, n_u64;
 885         int fields_size = 0;
 886
 887         event = trace_file->event_call->data;
 888
 889         if (trace_trigger_soft_disabled(trace_file))
 890                 return;
 891
 892         fields_size = event->n_u64 * sizeof(u64);
 893
 894         /*
 895          * Avoid ring buffer recursion detection, as this event
 896          * is being performed within another event.
 897          */
 898         buffer = trace_file->tr->array_buffer.buffer;
 899         ring_buffer_nest_start(buffer);
 900
 901         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 902                                            sizeof(*entry) + fields_size);
 903         if (!entry)
 904                 goto out;
 905
 906         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 907                 if (event->fields[i]->is_string) {
 908                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 909                         char *str_field = (char *)&entry->fields[n_u64];
 910
 911                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 912                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 913                 } else {
 914                         struct synth_field *field = event->fields[i];
 915                         u64 val = var_ref_vals[var_ref_idx + i];
 916
 917                         switch (field->size) {
 918                         case 1:
 919                                 *(u8 *)&entry->fields[n_u64] = (u8)val;
 920                                 break;
 921
 922                         case 2:
 923                                 *(u16 *)&entry->fields[n_u64] = (u16)val;
 924                                 break;
 925
 926                         case 4:
 927                                 *(u32 *)&entry->fields[n_u64] = (u32)val;
 928                                 break;
 929
 930                         default:
 931                                 entry->fields[n_u64] = val;
 932                                 break;
 933                         }
 934                         n_u64++;
 935                 }
 936         }
 937
 938         trace_event_buffer_commit(&fbuffer);
 939 out:
 940         ring_buffer_nest_end(buffer);
 941 }
 942
 943 static void free_synth_event_print_fmt(struct trace_event_call *call)
 944 {
 945         if (call) {
 946                 kfree(call->print_fmt);
 947                 call->print_fmt = NULL;
 948         }
 949 }
 950
 951 static int __set_synth_event_print_fmt(struct synth_event *event,
 952                                        char *buf, int len)
 953 {
 954         const char *fmt;
 955         int pos = 0;
 956         int i;
 957
 958         /* When len=0, we just calculate the needed length */
 959 #define LEN_OR_ZERO (len ? len - pos : 0)
 960
 961         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 962         for (i = 0; i < event->n_fields; i++) {
 963                 fmt = synth_field_fmt(event->fields[i]->type);
 964                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 965                                 event->fields[i]->name, fmt,
 966                                 i == event->n_fields - 1 ? "" : ", ");
 967         }
 968         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 969
 970         for (i = 0; i < event->n_fields; i++) {
 971                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 972                                 ", REC->%s", event->fields[i]->name);
 973         }
 974
 975 #undef LEN_OR_ZERO
 976
 977         /* return the length of print_fmt */
 978         return pos;
 979 }
 980
 981 static int set_synth_event_print_fmt(struct trace_event_call *call)
 982 {
 983         struct synth_event *event = call->data;
 984         char *print_fmt;
 985         int len;
 986
 987         /* First: called with 0 length to calculate the needed length */
 988         len = __set_synth_event_print_fmt(event, NULL, 0);
 989
 990         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 991         if (!print_fmt)
 992                 return -ENOMEM;
 993
 994         /* Second: actually write the @print_fmt */
 995         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 996         call->print_fmt = print_fmt;
 997
 998         return 0;
 999 }
1000
1001 static void free_synth_field(struct synth_field *field)
1002 {
1003         kfree(field->type);
1004         kfree(field->name);
1005         kfree(field);
1006 }
1007
1008 static struct synth_field *parse_synth_field(int argc, const char **argv,
1009                                              int *consumed)
1010 {
1011         struct synth_field *field;
1012         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
1013         int len, ret = 0;
1014
1015         if (field_type[0] == ';')
1016                 field_type++;
1017
1018         if (!strcmp(field_type, "unsigned")) {
1019                 if (argc < 3)
1020                         return ERR_PTR(-EINVAL);
1021                 prefix = "unsigned ";
1022                 field_type = argv[1];
1023                 field_name = argv[2];
1024                 *consumed = 3;
1025         } else {
1026                 field_name = argv[1];
1027                 *consumed = 2;
1028         }
1029
1030         field = kzalloc(sizeof(*field), GFP_KERNEL);
1031         if (!field)
1032                 return ERR_PTR(-ENOMEM);
1033
1034         len = strlen(field_name);
1035         array = strchr(field_name, '[');
1036         if (array)
1037                 len -= strlen(array);
1038         else if (field_name[len - 1] == ';')
1039                 len--;
1040
1041         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1042         if (!field->name) {
1043                 ret = -ENOMEM;
1044                 goto free;
1045         }
1046
1047         if (field_type[0] == ';')
1048                 field_type++;
1049         len = strlen(field_type) + 1;
1050         if (array)
1051                 len += strlen(array);
1052         if (prefix)
1053                 len += strlen(prefix);
1054
1055         field->type = kzalloc(len, GFP_KERNEL);
1056         if (!field->type) {
1057                 ret = -ENOMEM;
1058                 goto free;
1059         }
1060         if (prefix)
1061                 strcat(field->type, prefix);
1062         strcat(field->type, field_type);
1063         if (array) {
1064                 strcat(field->type, array);
1065                 if (field->type[len - 1] == ';')
1066                         field->type[len - 1] = '\0';
1067         }
1068
1069         field->size = synth_field_size(field->type);
1070         if (!field->size) {
1071                 ret = -EINVAL;
1072                 goto free;
1073         }
1074
1075         if (synth_field_is_string(field->type))
1076                 field->is_string = true;
1077
1078         field->is_signed = synth_field_signed(field->type);
1079
1080  out:
1081         return field;
1082  free:
1083         free_synth_field(field);
1084         field = ERR_PTR(ret);
1085         goto out;
1086 }
1087
1088 static void free_synth_tracepoint(struct tracepoint *tp)
1089 {
1090         if (!tp)
1091                 return;
1092
1093         kfree(tp->name);
1094         kfree(tp);
1095 }
1096
1097 static struct tracepoint *alloc_synth_tracepoint(char *name)
1098 {
1099         struct tracepoint *tp;
1100
1101         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1102         if (!tp)
1103                 return ERR_PTR(-ENOMEM);
1104
1105         tp->name = kstrdup(name, GFP_KERNEL);
1106         if (!tp->name) {
1107                 kfree(tp);
1108                 return ERR_PTR(-ENOMEM);
1109         }
1110
1111         return tp;
1112 }
1113
1114 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1115                                     unsigned int var_ref_idx);
1116
1117 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1118                                unsigned int var_ref_idx)
1119 {
1120         struct tracepoint *tp = event->tp;
1121
1122         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1123                 struct tracepoint_func *probe_func_ptr;
1124                 synth_probe_func_t probe_func;
1125                 void *__data;
1126
1127                 if (!(cpu_online(raw_smp_processor_id())))
1128                         return;
1129
1130                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1131                 if (probe_func_ptr) {
1132                         do {
1133                                 probe_func = probe_func_ptr->func;
1134                                 __data = probe_func_ptr->data;
1135                                 probe_func(__data, var_ref_vals, var_ref_idx);
1136                         } while ((++probe_func_ptr)->func);
1137                 }
1138         }
1139 }
1140
1141 static struct synth_event *find_synth_event(const char *name)
1142 {
1143         struct dyn_event *pos;
1144         struct synth_event *event;
1145
1146         for_each_dyn_event(pos) {
1147                 if (!is_synth_event(pos))
1148                         continue;
1149                 event = to_synth_event(pos);
1150                 if (strcmp(event->name, name) == 0)
1151                         return event;
1152         }
1153
1154         return NULL;
1155 }
1156
1157 static int register_synth_event(struct synth_event *event)
1158 {
1159         struct trace_event_call *call = &event->call;
1160         int ret = 0;
1161
1162         event->call.class = &event->class;
1163         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1164         if (!event->class.system) {
1165                 ret = -ENOMEM;
1166                 goto out;
1167         }
1168
1169         event->tp = alloc_synth_tracepoint(event->name);
1170         if (IS_ERR(event->tp)) {
1171                 ret = PTR_ERR(event->tp);
1172                 event->tp = NULL;
1173                 goto out;
1174         }
1175
1176         INIT_LIST_HEAD(&call->class->fields);
1177         call->event.funcs = &synth_event_funcs;
1178         call->class->define_fields = synth_event_define_fields;
1179
1180         ret = register_trace_event(&call->event);
1181         if (!ret) {
1182                 ret = -ENODEV;
1183                 goto out;
1184         }
1185         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1186         call->class->reg = trace_event_reg;
1187         call->class->probe = trace_event_raw_event_synth;
1188         call->data = event;
1189         call->tp = event->tp;
1190
1191         ret = trace_add_event_call(call);
1192         if (ret) {
1193                 pr_warn("Failed to register synthetic event: %s\n",
1194                         trace_event_name(call));
1195                 goto err;
1196         }
1197
1198         ret = set_synth_event_print_fmt(call);
1199         if (ret < 0) {
1200                 trace_remove_event_call(call);
1201                 goto err;
1202         }
1203  out:
1204         return ret;
1205  err:
1206         unregister_trace_event(&call->event);
1207         goto out;
1208 }
1209
1210 static int unregister_synth_event(struct synth_event *event)
1211 {
1212         struct trace_event_call *call = &event->call;
1213         int ret;
1214
1215         ret = trace_remove_event_call(call);
1216
1217         return ret;
1218 }
1219
1220 static void free_synth_event(struct synth_event *event)
1221 {
1222         unsigned int i;
1223
1224         if (!event)
1225                 return;
1226
1227         for (i = 0; i < event->n_fields; i++)
1228                 free_synth_field(event->fields[i]);
1229
1230         kfree(event->fields);
1231         kfree(event->name);
1232         kfree(event->class.system);
1233         free_synth_tracepoint(event->tp);
1234         free_synth_event_print_fmt(&event->call);
1235         kfree(event);
1236 }
1237
1238 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1239                                              struct synth_field **fields)
1240 {
1241         struct synth_event *event;
1242         unsigned int i;
1243
1244         event = kzalloc(sizeof(*event), GFP_KERNEL);
1245         if (!event) {
1246                 event = ERR_PTR(-ENOMEM);
1247                 goto out;
1248         }
1249
1250         event->name = kstrdup(name, GFP_KERNEL);
1251         if (!event->name) {
1252                 kfree(event);
1253                 event = ERR_PTR(-ENOMEM);
1254                 goto out;
1255         }
1256
1257         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1258         if (!event->fields) {
1259                 free_synth_event(event);
1260                 event = ERR_PTR(-ENOMEM);
1261                 goto out;
1262         }
1263
1264         dyn_event_init(&event->devent, &synth_event_ops);
1265
1266         for (i = 0; i < n_fields; i++)
1267                 event->fields[i] = fields[i];
1268
1269         event->n_fields = n_fields;
1270  out:
1271         return event;
1272 }
1273
1274 static void action_trace(struct hist_trigger_data *hist_data,
1275                          struct tracing_map_elt *elt, void *rec,
1276                          struct ring_buffer_event *rbe, void *key,
1277                          struct action_data *data, u64 *var_ref_vals)
1278 {
1279         struct synth_event *event = data->synth_event;
1280
1281         trace_synth(event, var_ref_vals, data->var_ref_idx);
1282 }
1283
1284 struct hist_var_data {
1285         struct list_head list;
1286         struct hist_trigger_data *hist_data;
1287 };
1288
1289 static int __create_synth_event(int argc, const char *name, const char **argv)
1290 {
1291         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1292         struct synth_event *event = NULL;
1293         int i, consumed = 0, n_fields = 0, ret = 0;
1294
1295         /*
1296          * Argument syntax:
1297          *  - Add synthetic event: <event_name> field[;field] ...
1298          *  - Remove synthetic event: !<event_name> field[;field] ...
1299          *      where 'field' = type field_name
1300          */
1301
1302         if (name[0] == '\0' || argc < 1)
1303                 return -EINVAL;
1304
1305         mutex_lock(&event_mutex);
1306
1307         event = find_synth_event(name);
1308         if (event) {
1309                 ret = -EEXIST;
1310                 goto out;
1311         }
1312
1313         for (i = 0; i < argc - 1; i++) {
1314                 if (strcmp(argv[i], ";") == 0)
1315                         continue;
1316                 if (n_fields == SYNTH_FIELDS_MAX) {
1317                         ret = -EINVAL;
1318                         goto err;
1319                 }
1320
1321                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1322                 if (IS_ERR(field)) {
1323                         ret = PTR_ERR(field);
1324                         goto err;
1325                 }
1326                 fields[n_fields++] = field;
1327                 i += consumed - 1;
1328         }
1329
1330         if (i < argc && strcmp(argv[i], ";") != 0) {
1331                 ret = -EINVAL;
1332                 goto err;
1333         }
1334
1335         event = alloc_synth_event(name, n_fields, fields);
1336         if (IS_ERR(event)) {
1337                 ret = PTR_ERR(event);
1338                 event = NULL;
1339                 goto err;
1340         }
1341         ret = register_synth_event(event);
1342         if (!ret)
1343                 dyn_event_add(&event->devent);
1344         else
1345                 free_synth_event(event);
1346  out:
1347         mutex_unlock(&event_mutex);
1348
1349         return ret;
1350  err:
1351         for (i = 0; i < n_fields; i++)
1352                 free_synth_field(fields[i]);
1353
1354         goto out;
1355 }
1356
1357 static int create_or_delete_synth_event(int argc, char **argv)
1358 {
1359         const char *name = argv[0];
1360         struct synth_event *event = NULL;
1361         int ret;
1362
1363         /* trace_run_command() ensures argc != 0 */
1364         if (name[0] == '!') {
1365                 mutex_lock(&event_mutex);
1366                 event = find_synth_event(name + 1);
1367                 if (event) {
1368                         if (event->ref)
1369                                 ret = -EBUSY;
1370                         else {
1371                                 ret = unregister_synth_event(event);
1372                                 if (!ret) {
1373                                         dyn_event_remove(&event->devent);
1374                                         free_synth_event(event);
1375                                 }
1376                         }
1377                 } else
1378                         ret = -ENOENT;
1379                 mutex_unlock(&event_mutex);
1380                 return ret;
1381         }
1382
1383         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1384         return ret == -ECANCELED ? -EINVAL : ret;
1385 }
1386
1387 int synth_event_run_command(const char *command)
1388 {
1389         return trace_run_command(command, create_or_delete_synth_event);
1390 }
1391
1392 static int synth_event_create(int argc, const char **argv)
1393 {
1394         const char *name = argv[0];
1395         int len;
1396
1397         if (name[0] != 's' || name[1] != ':')
1398                 return -ECANCELED;
1399         name += 2;
1400
1401         /* This interface accepts group name prefix */
1402         if (strchr(name, '/')) {
1403                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
1404                 if (len == 0)
1405                         return -EINVAL;
1406                 name += len;
1407         }
1408         return __create_synth_event(argc - 1, name, argv + 1);
1409 }
1410
1411 static int synth_event_release(struct dyn_event *ev)
1412 {
1413         struct synth_event *event = to_synth_event(ev);
1414         int ret;
1415
1416         if (event->ref)
1417                 return -EBUSY;
1418
1419         ret = unregister_synth_event(event);
1420         if (ret)
1421                 return ret;
1422
1423         dyn_event_remove(ev);
1424         free_synth_event(event);
1425         return 0;
1426 }
1427
1428 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1429 {
1430         struct synth_field *field;
1431         unsigned int i;
1432
1433         seq_printf(m, "%s\t", event->name);
1434
1435         for (i = 0; i < event->n_fields; i++) {
1436                 field = event->fields[i];
1437
1438                 /* parameter values */
1439                 seq_printf(m, "%s %s%s", field->type, field->name,
1440                            i == event->n_fields - 1 ? "" : "; ");
1441         }
1442
1443         seq_putc(m, '\n');
1444
1445         return 0;
1446 }
1447
1448 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1449 {
1450         struct synth_event *event = to_synth_event(ev);
1451
1452         seq_printf(m, "s:%s/", event->class.system);
1453
1454         return __synth_event_show(m, event);
1455 }
1456
1457 static int synth_events_seq_show(struct seq_file *m, void *v)
1458 {
1459         struct dyn_event *ev = v;
1460
1461         if (!is_synth_event(ev))
1462                 return 0;
1463
1464         return __synth_event_show(m, to_synth_event(ev));
1465 }
1466
1467 static const struct seq_operations synth_events_seq_op = {
1468         .start  = dyn_event_seq_start,
1469         .next   = dyn_event_seq_next,
1470         .stop   = dyn_event_seq_stop,
1471         .show   = synth_events_seq_show,
1472 };
1473
1474 static int synth_events_open(struct inode *inode, struct file *file)
1475 {
1476         int ret;
1477
1478         ret = security_locked_down(LOCKDOWN_TRACEFS);
1479         if (ret)
1480                 return ret;
1481
1482         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1483                 ret = dyn_events_release_all(&synth_event_ops);
1484                 if (ret < 0)
1485                         return ret;
1486         }
1487
1488         return seq_open(file, &synth_events_seq_op);
1489 }
1490
1491 static ssize_t synth_events_write(struct file *file,
1492                                   const char __user *buffer,
1493                                   size_t count, loff_t *ppos)
1494 {
1495         return trace_parse_run_command(file, buffer, count, ppos,
1496                                        create_or_delete_synth_event);
1497 }
1498
1499 static const struct file_operations synth_events_fops = {
1500         .open           = synth_events_open,
1501         .write          = synth_events_write,
1502         .read           = seq_read,
1503         .llseek         = seq_lseek,
1504         .release        = seq_release,
1505 };
1506
1507 static u64 hist_field_timestamp(struct hist_field *hist_field,
1508                                 struct tracing_map_elt *elt,
1509                                 struct ring_buffer_event *rbe,
1510                                 void *event)
1511 {
1512         struct hist_trigger_data *hist_data = hist_field->hist_data;
1513         struct trace_array *tr = hist_data->event_file->tr;
1514
1515         u64 ts = ring_buffer_event_time_stamp(rbe);
1516
1517         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1518                 ts = ns2usecs(ts);
1519
1520         return ts;
1521 }
1522
1523 static u64 hist_field_cpu(struct hist_field *hist_field,
1524                           struct tracing_map_elt *elt,
1525                           struct ring_buffer_event *rbe,
1526                           void *event)
1527 {
1528         int cpu = smp_processor_id();
1529
1530         return cpu;
1531 }
1532
1533 /**
1534  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1535  * @hist_field: The VAR_REF field to check
1536  * @var_data: The hist trigger that owns the variable
1537  * @var_idx: The trigger variable identifier
1538  *
1539  * Check the given VAR_REF field to see whether or not it references
1540  * the given variable associated with the given trigger.
1541  *
1542  * Return: The VAR_REF field if it does reference the variable, NULL if not
1543  */
1544 static struct hist_field *
1545 check_field_for_var_ref(struct hist_field *hist_field,
1546                         struct hist_trigger_data *var_data,
1547                         unsigned int var_idx)
1548 {
1549         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1550
1551         if (hist_field && hist_field->var.idx == var_idx &&
1552             hist_field->var.hist_data == var_data)
1553                 return hist_field;
1554
1555         return NULL;
1556 }
1557
1558 /**
1559  * find_var_ref - Check if a trigger has a reference to a trigger variable
1560  * @hist_data: The hist trigger that might have a reference to the variable
1561  * @var_data: The hist trigger that owns the variable
1562  * @var_idx: The trigger variable identifier
1563  *
1564  * Check the list of var_refs[] on the first hist trigger to see
1565  * whether any of them are references to the variable on the second
1566  * trigger.
1567  *
1568  * Return: The VAR_REF field referencing the variable if so, NULL if not
1569  */
1570 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1571                                        struct hist_trigger_data *var_data,
1572                                        unsigned int var_idx)
1573 {
1574         struct hist_field *hist_field;
1575         unsigned int i;
1576
1577         for (i = 0; i < hist_data->n_var_refs; i++) {
1578                 hist_field = hist_data->var_refs[i];
1579                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1580                         return hist_field;
1581         }
1582
1583         return NULL;
1584 }
1585
1586 /**
1587  * find_any_var_ref - Check if there is a reference to a given trigger variable
1588  * @hist_data: The hist trigger
1589  * @var_idx: The trigger variable identifier
1590  *
1591  * Check to see whether the given variable is currently referenced by
1592  * any other trigger.
1593  *
1594  * The trigger the variable is defined on is explicitly excluded - the
1595  * assumption being that a self-reference doesn't prevent a trigger
1596  * from being removed.
1597  *
1598  * Return: The VAR_REF field referencing the variable if so, NULL if not
1599  */
1600 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1601                                            unsigned int var_idx)
1602 {
1603         struct trace_array *tr = hist_data->event_file->tr;
1604         struct hist_field *found = NULL;
1605         struct hist_var_data *var_data;
1606
1607         list_for_each_entry(var_data, &tr->hist_vars, list) {
1608                 if (var_data->hist_data == hist_data)
1609                         continue;
1610                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1611                 if (found)
1612                         break;
1613         }
1614
1615         return found;
1616 }
1617
1618 /**
1619  * check_var_refs - Check if there is a reference to any of trigger's variables
1620  * @hist_data: The hist trigger
1621  *
1622  * A trigger can define one or more variables.  If any one of them is
1623  * currently referenced by any other trigger, this function will
1624  * determine that.
1625
1626  * Typically used to determine whether or not a trigger can be removed
1627  * - if there are any references to a trigger's variables, it cannot.
1628  *
1629  * Return: True if there is a reference to any of trigger's variables
1630  */
1631 static bool check_var_refs(struct hist_trigger_data *hist_data)
1632 {
1633         struct hist_field *field;
1634         bool found = false;
1635         int i;
1636
1637         for_each_hist_field(i, hist_data) {
1638                 field = hist_data->fields[i];
1639                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1640                         if (find_any_var_ref(hist_data, field->var.idx)) {
1641                                 found = true;
1642                                 break;
1643                         }
1644                 }
1645         }
1646
1647         return found;
1648 }
1649
1650 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1651 {
1652         struct trace_array *tr = hist_data->event_file->tr;
1653         struct hist_var_data *var_data, *found = NULL;
1654
1655         list_for_each_entry(var_data, &tr->hist_vars, list) {
1656                 if (var_data->hist_data == hist_data) {
1657                         found = var_data;
1658                         break;
1659                 }
1660         }
1661
1662         return found;
1663 }
1664
1665 static bool field_has_hist_vars(struct hist_field *hist_field,
1666                                 unsigned int level)
1667 {
1668         int i;
1669
1670         if (level > 3)
1671                 return false;
1672
1673         if (!hist_field)
1674                 return false;
1675
1676         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1677             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1678                 return true;
1679
1680         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1681                 struct hist_field *operand;
1682
1683                 operand = hist_field->operands[i];
1684                 if (field_has_hist_vars(operand, level + 1))
1685                         return true;
1686         }
1687
1688         return false;
1689 }
1690
1691 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1692 {
1693         struct hist_field *hist_field;
1694         int i;
1695
1696         for_each_hist_field(i, hist_data) {
1697                 hist_field = hist_data->fields[i];
1698                 if (field_has_hist_vars(hist_field, 0))
1699                         return true;
1700         }
1701
1702         return false;
1703 }
1704
1705 static int save_hist_vars(struct hist_trigger_data *hist_data)
1706 {
1707         struct trace_array *tr = hist_data->event_file->tr;
1708         struct hist_var_data *var_data;
1709
1710         var_data = find_hist_vars(hist_data);
1711         if (var_data)
1712                 return 0;
1713
1714         if (tracing_check_open_get_tr(tr))
1715                 return -ENODEV;
1716
1717         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1718         if (!var_data) {
1719                 trace_array_put(tr);
1720                 return -ENOMEM;
1721         }
1722
1723         var_data->hist_data = hist_data;
1724         list_add(&var_data->list, &tr->hist_vars);
1725
1726         return 0;
1727 }
1728
1729 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1730 {
1731         struct trace_array *tr = hist_data->event_file->tr;
1732         struct hist_var_data *var_data;
1733
1734         var_data = find_hist_vars(hist_data);
1735         if (!var_data)
1736                 return;
1737
1738         if (WARN_ON(check_var_refs(hist_data)))
1739                 return;
1740
1741         list_del(&var_data->list);
1742
1743         kfree(var_data);
1744
1745         trace_array_put(tr);
1746 }
1747
1748 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1749                                          const char *var_name)
1750 {
1751         struct hist_field *hist_field, *found = NULL;
1752         int i;
1753
1754         for_each_hist_field(i, hist_data) {
1755                 hist_field = hist_data->fields[i];
1756                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1757                     strcmp(hist_field->var.name, var_name) == 0) {
1758                         found = hist_field;
1759                         break;
1760                 }
1761         }
1762
1763         return found;
1764 }
1765
1766 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1767                                    struct trace_event_file *file,
1768                                    const char *var_name)
1769 {
1770         struct hist_trigger_data *test_data;
1771         struct event_trigger_data *test;
1772         struct hist_field *hist_field;
1773
1774         lockdep_assert_held(&event_mutex);
1775
1776         hist_field = find_var_field(hist_data, var_name);
1777         if (hist_field)
1778                 return hist_field;
1779
1780         list_for_each_entry(test, &file->triggers, list) {
1781                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1782                         test_data = test->private_data;
1783                         hist_field = find_var_field(test_data, var_name);
1784                         if (hist_field)
1785                                 return hist_field;
1786                 }
1787         }
1788
1789         return NULL;
1790 }
1791
1792 static struct trace_event_file *find_var_file(struct trace_array *tr,
1793                                               char *system,
1794                                               char *event_name,
1795                                               char *var_name)
1796 {
1797         struct hist_trigger_data *var_hist_data;
1798         struct hist_var_data *var_data;
1799         struct trace_event_file *file, *found = NULL;
1800
1801         if (system)
1802                 return find_event_file(tr, system, event_name);
1803
1804         list_for_each_entry(var_data, &tr->hist_vars, list) {
1805                 var_hist_data = var_data->hist_data;
1806                 file = var_hist_data->event_file;
1807                 if (file == found)
1808                         continue;
1809
1810                 if (find_var_field(var_hist_data, var_name)) {
1811                         if (found) {
1812                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1813                                 return NULL;
1814                         }
1815
1816                         found = file;
1817                 }
1818         }
1819
1820         return found;
1821 }
1822
1823 static struct hist_field *find_file_var(struct trace_event_file *file,
1824                                         const char *var_name)
1825 {
1826         struct hist_trigger_data *test_data;
1827         struct event_trigger_data *test;
1828         struct hist_field *hist_field;
1829
1830         lockdep_assert_held(&event_mutex);
1831
1832         list_for_each_entry(test, &file->triggers, list) {
1833                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1834                         test_data = test->private_data;
1835                         hist_field = find_var_field(test_data, var_name);
1836                         if (hist_field)
1837                                 return hist_field;
1838                 }
1839         }
1840
1841         return NULL;
1842 }
1843
1844 static struct hist_field *
1845 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1846 {
1847         struct trace_array *tr = hist_data->event_file->tr;
1848         struct hist_field *hist_field, *found = NULL;
1849         struct trace_event_file *file;
1850         unsigned int i;
1851
1852         for (i = 0; i < hist_data->n_actions; i++) {
1853                 struct action_data *data = hist_data->actions[i];
1854
1855                 if (data->handler == HANDLER_ONMATCH) {
1856                         char *system = data->match_data.event_system;
1857                         char *event_name = data->match_data.event;
1858
1859                         file = find_var_file(tr, system, event_name, var_name);
1860                         if (!file)
1861                                 continue;
1862                         hist_field = find_file_var(file, var_name);
1863                         if (hist_field) {
1864                                 if (found) {
1865                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1866                                                  errpos(var_name));
1867                                         return ERR_PTR(-EINVAL);
1868                                 }
1869
1870                                 found = hist_field;
1871                         }
1872                 }
1873         }
1874         return found;
1875 }
1876
1877 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1878                                          char *system,
1879                                          char *event_name,
1880                                          char *var_name)
1881 {
1882         struct trace_array *tr = hist_data->event_file->tr;
1883         struct hist_field *hist_field = NULL;
1884         struct trace_event_file *file;
1885
1886         if (!system || !event_name) {
1887                 hist_field = find_match_var(hist_data, var_name);
1888                 if (IS_ERR(hist_field))
1889                         return NULL;
1890                 if (hist_field)
1891                         return hist_field;
1892         }
1893
1894         file = find_var_file(tr, system, event_name, var_name);
1895         if (!file)
1896                 return NULL;
1897
1898         hist_field = find_file_var(file, var_name);
1899
1900         return hist_field;
1901 }
1902
1903 static u64 hist_field_var_ref(struct hist_field *hist_field,
1904                               struct tracing_map_elt *elt,
1905                               struct ring_buffer_event *rbe,
1906                               void *event)
1907 {
1908         struct hist_elt_data *elt_data;
1909         u64 var_val = 0;
1910
1911         if (WARN_ON_ONCE(!elt))
1912                 return var_val;
1913
1914         elt_data = elt->private_data;
1915         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1916
1917         return var_val;
1918 }
1919
1920 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1921                              u64 *var_ref_vals, bool self)
1922 {
1923         struct hist_trigger_data *var_data;
1924         struct tracing_map_elt *var_elt;
1925         struct hist_field *hist_field;
1926         unsigned int i, var_idx;
1927         bool resolved = true;
1928         u64 var_val = 0;
1929
1930         for (i = 0; i < hist_data->n_var_refs; i++) {
1931                 hist_field = hist_data->var_refs[i];
1932                 var_idx = hist_field->var.idx;
1933                 var_data = hist_field->var.hist_data;
1934
1935                 if (var_data == NULL) {
1936                         resolved = false;
1937                         break;
1938                 }
1939
1940                 if ((self && var_data != hist_data) ||
1941                     (!self && var_data == hist_data))
1942                         continue;
1943
1944                 var_elt = tracing_map_lookup(var_data->map, key);
1945                 if (!var_elt) {
1946                         resolved = false;
1947                         break;
1948                 }
1949
1950                 if (!tracing_map_var_set(var_elt, var_idx)) {
1951                         resolved = false;
1952                         break;
1953                 }
1954
1955                 if (self || !hist_field->read_once)
1956                         var_val = tracing_map_read_var(var_elt, var_idx);
1957                 else
1958                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1959
1960                 var_ref_vals[i] = var_val;
1961         }
1962
1963         return resolved;
1964 }
1965
1966 static const char *hist_field_name(struct hist_field *field,
1967                                    unsigned int level)
1968 {
1969         const char *field_name = "";
1970
1971         if (level > 1)
1972                 return field_name;
1973
1974         if (field->field)
1975                 field_name = field->field->name;
1976         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1977                  field->flags & HIST_FIELD_FL_ALIAS)
1978                 field_name = hist_field_name(field->operands[0], ++level);
1979         else if (field->flags & HIST_FIELD_FL_CPU)
1980                 field_name = "cpu";
1981         else if (field->flags & HIST_FIELD_FL_EXPR ||
1982                  field->flags & HIST_FIELD_FL_VAR_REF) {
1983                 if (field->system) {
1984                         static char full_name[MAX_FILTER_STR_VAL];
1985
1986                         strcat(full_name, field->system);
1987                         strcat(full_name, ".");
1988                         strcat(full_name, field->event_name);
1989                         strcat(full_name, ".");
1990                         strcat(full_name, field->name);
1991                         field_name = full_name;
1992                 } else
1993                         field_name = field->name;
1994         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1995                 field_name = "common_timestamp";
1996
1997         if (field_name == NULL)
1998                 field_name = "";
1999
2000         return field_name;
2001 }
2002
2003 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
2004 {
2005         hist_field_fn_t fn = NULL;
2006
2007         switch (field_size) {
2008         case 8:
2009                 if (field_is_signed)
2010                         fn = hist_field_s64;
2011                 else
2012                         fn = hist_field_u64;
2013                 break;
2014         case 4:
2015                 if (field_is_signed)
2016                         fn = hist_field_s32;
2017                 else
2018                         fn = hist_field_u32;
2019                 break;
2020         case 2:
2021                 if (field_is_signed)
2022                         fn = hist_field_s16;
2023                 else
2024                         fn = hist_field_u16;
2025                 break;
2026         case 1:
2027                 if (field_is_signed)
2028                         fn = hist_field_s8;
2029                 else
2030                         fn = hist_field_u8;
2031                 break;
2032         }
2033
2034         return fn;
2035 }
2036
2037 static int parse_map_size(char *str)
2038 {
2039         unsigned long size, map_bits;
2040         int ret;
2041
2042         ret = kstrtoul(str, 0, &size);
2043         if (ret)
2044                 goto out;
2045
2046         map_bits = ilog2(roundup_pow_of_two(size));
2047         if (map_bits < TRACING_MAP_BITS_MIN ||
2048             map_bits > TRACING_MAP_BITS_MAX)
2049                 ret = -EINVAL;
2050         else
2051                 ret = map_bits;
2052  out:
2053         return ret;
2054 }
2055
2056 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2057 {
2058         unsigned int i;
2059
2060         if (!attrs)
2061                 return;
2062
2063         for (i = 0; i < attrs->n_assignments; i++)
2064                 kfree(attrs->assignment_str[i]);
2065
2066         for (i = 0; i < attrs->n_actions; i++)
2067                 kfree(attrs->action_str[i]);
2068
2069         kfree(attrs->name);
2070         kfree(attrs->sort_key_str);
2071         kfree(attrs->keys_str);
2072         kfree(attrs->vals_str);
2073         kfree(attrs->clock);
2074         kfree(attrs);
2075 }
2076
2077 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2078 {
2079         int ret = -EINVAL;
2080
2081         if (attrs->n_actions >= HIST_ACTIONS_MAX)
2082                 return ret;
2083
2084         if ((str_has_prefix(str, "onmatch(")) ||
2085             (str_has_prefix(str, "onmax(")) ||
2086             (str_has_prefix(str, "onchange("))) {
2087                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2088                 if (!attrs->action_str[attrs->n_actions]) {
2089                         ret = -ENOMEM;
2090                         return ret;
2091                 }
2092                 attrs->n_actions++;
2093                 ret = 0;
2094         }
2095         return ret;
2096 }
2097
2098 static int parse_assignment(struct trace_array *tr,
2099                             char *str, struct hist_trigger_attrs *attrs)
2100 {
2101         int len, ret = 0;
2102
2103         if ((len = str_has_prefix(str, "key=")) ||
2104             (len = str_has_prefix(str, "keys="))) {
2105                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
2106                 if (!attrs->keys_str) {
2107                         ret = -ENOMEM;
2108                         goto out;
2109                 }
2110         } else if ((len = str_has_prefix(str, "val=")) ||
2111                    (len = str_has_prefix(str, "vals=")) ||
2112                    (len = str_has_prefix(str, "values="))) {
2113                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
2114                 if (!attrs->vals_str) {
2115                         ret = -ENOMEM;
2116                         goto out;
2117                 }
2118         } else if ((len = str_has_prefix(str, "sort="))) {
2119                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
2120                 if (!attrs->sort_key_str) {
2121                         ret = -ENOMEM;
2122                         goto out;
2123                 }
2124         } else if (str_has_prefix(str, "name=")) {
2125                 attrs->name = kstrdup(str, GFP_KERNEL);
2126                 if (!attrs->name) {
2127                         ret = -ENOMEM;
2128                         goto out;
2129                 }
2130         } else if ((len = str_has_prefix(str, "clock="))) {
2131                 str += len;
2132
2133                 str = strstrip(str);
2134                 attrs->clock = kstrdup(str, GFP_KERNEL);
2135                 if (!attrs->clock) {
2136                         ret = -ENOMEM;
2137                         goto out;
2138                 }
2139         } else if ((len = str_has_prefix(str, "size="))) {
2140                 int map_bits = parse_map_size(str + len);
2141
2142                 if (map_bits < 0) {
2143                         ret = map_bits;
2144                         goto out;
2145                 }
2146                 attrs->map_bits = map_bits;
2147         } else {
2148                 char *assignment;
2149
2150                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2151                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
2152                         ret = -EINVAL;
2153                         goto out;
2154                 }
2155
2156                 assignment = kstrdup(str, GFP_KERNEL);
2157                 if (!assignment) {
2158                         ret = -ENOMEM;
2159                         goto out;
2160                 }
2161
2162                 attrs->assignment_str[attrs->n_assignments++] = assignment;
2163         }
2164  out:
2165         return ret;
2166 }
2167
2168 static struct hist_trigger_attrs *
2169 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
2170 {
2171         struct hist_trigger_attrs *attrs;
2172         int ret = 0;
2173
2174         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2175         if (!attrs)
2176                 return ERR_PTR(-ENOMEM);
2177
2178         while (trigger_str) {
2179                 char *str = strsep(&trigger_str, ":");
2180                 char *rhs;
2181
2182                 rhs = strchr(str, '=');
2183                 if (rhs) {
2184                         if (!strlen(++rhs)) {
2185                                 ret = -EINVAL;
2186                                 goto free;
2187                         }
2188                         ret = parse_assignment(tr, str, attrs);
2189                         if (ret)
2190                                 goto free;
2191                 } else if (strcmp(str, "pause") == 0)
2192                         attrs->pause = true;
2193                 else if ((strcmp(str, "cont") == 0) ||
2194                          (strcmp(str, "continue") == 0))
2195                         attrs->cont = true;
2196                 else if (strcmp(str, "clear") == 0)
2197                         attrs->clear = true;
2198                 else {
2199                         ret = parse_action(str, attrs);
2200                         if (ret)
2201                                 goto free;
2202                 }
2203         }
2204
2205         if (!attrs->keys_str) {
2206                 ret = -EINVAL;
2207                 goto free;
2208         }
2209
2210         if (!attrs->clock) {
2211                 attrs->clock = kstrdup("global", GFP_KERNEL);
2212                 if (!attrs->clock) {
2213                         ret = -ENOMEM;
2214                         goto free;
2215                 }
2216         }
2217
2218         return attrs;
2219  free:
2220         destroy_hist_trigger_attrs(attrs);
2221
2222         return ERR_PTR(ret);
2223 }
2224
2225 static inline void save_comm(char *comm, struct task_struct *task)
2226 {
2227         if (!task->pid) {
2228                 strcpy(comm, "<idle>");
2229                 return;
2230         }
2231
2232         if (WARN_ON_ONCE(task->pid < 0)) {
2233                 strcpy(comm, "<XXX>");
2234                 return;
2235         }
2236
2237         strncpy(comm, task->comm, TASK_COMM_LEN);
2238 }
2239
2240 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2241 {
2242         unsigned int i;
2243
2244         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2245                 kfree(elt_data->field_var_str[i]);
2246
2247         kfree(elt_data->comm);
2248         kfree(elt_data);
2249 }
2250
2251 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2252 {
2253         struct hist_elt_data *elt_data = elt->private_data;
2254
2255         hist_elt_data_free(elt_data);
2256 }
2257
2258 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2259 {
2260         struct hist_trigger_data *hist_data = elt->map->private_data;
2261         unsigned int size = TASK_COMM_LEN;
2262         struct hist_elt_data *elt_data;
2263         struct hist_field *key_field;
2264         unsigned int i, n_str;
2265
2266         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2267         if (!elt_data)
2268                 return -ENOMEM;
2269
2270         for_each_hist_key_field(i, hist_data) {
2271                 key_field = hist_data->fields[i];
2272
2273                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2274                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2275                         if (!elt_data->comm) {
2276                                 kfree(elt_data);
2277                                 return -ENOMEM;
2278                         }
2279                         break;
2280                 }
2281         }
2282
2283         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2284
2285         size = STR_VAR_LEN_MAX;
2286
2287         for (i = 0; i < n_str; i++) {
2288                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2289                 if (!elt_data->field_var_str[i]) {
2290                         hist_elt_data_free(elt_data);
2291                         return -ENOMEM;
2292                 }
2293         }
2294
2295         elt->private_data = elt_data;
2296
2297         return 0;
2298 }
2299
2300 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2301 {
2302         struct hist_elt_data *elt_data = elt->private_data;
2303
2304         if (elt_data->comm)
2305                 save_comm(elt_data->comm, current);
2306 }
2307
2308 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2309         .elt_alloc      = hist_trigger_elt_data_alloc,
2310         .elt_free       = hist_trigger_elt_data_free,
2311         .elt_init       = hist_trigger_elt_data_init,
2312 };
2313
2314 static const char *get_hist_field_flags(struct hist_field *hist_field)
2315 {
2316         const char *flags_str = NULL;
2317
2318         if (hist_field->flags & HIST_FIELD_FL_HEX)
2319                 flags_str = "hex";
2320         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2321                 flags_str = "sym";
2322         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2323                 flags_str = "sym-offset";
2324         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2325                 flags_str = "execname";
2326         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2327                 flags_str = "syscall";
2328         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2329                 flags_str = "log2";
2330         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2331                 flags_str = "usecs";
2332
2333         return flags_str;
2334 }
2335
2336 static void expr_field_str(struct hist_field *field, char *expr)
2337 {
2338         if (field->flags & HIST_FIELD_FL_VAR_REF)
2339                 strcat(expr, "$");
2340
2341         strcat(expr, hist_field_name(field, 0));
2342
2343         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2344                 const char *flags_str = get_hist_field_flags(field);
2345
2346                 if (flags_str) {
2347                         strcat(expr, ".");
2348                         strcat(expr, flags_str);
2349                 }
2350         }
2351 }
2352
2353 static char *expr_str(struct hist_field *field, unsigned int level)
2354 {
2355         char *expr;
2356
2357         if (level > 1)
2358                 return NULL;
2359
2360         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2361         if (!expr)
2362                 return NULL;
2363
2364         if (!field->operands[0]) {
2365                 expr_field_str(field, expr);
2366                 return expr;
2367         }
2368
2369         if (field->operator == FIELD_OP_UNARY_MINUS) {
2370                 char *subexpr;
2371
2372                 strcat(expr, "-(");
2373                 subexpr = expr_str(field->operands[0], ++level);
2374                 if (!subexpr) {
2375                         kfree(expr);
2376                         return NULL;
2377                 }
2378                 strcat(expr, subexpr);
2379                 strcat(expr, ")");
2380
2381                 kfree(subexpr);
2382
2383                 return expr;
2384         }
2385
2386         expr_field_str(field->operands[0], expr);
2387
2388         switch (field->operator) {
2389         case FIELD_OP_MINUS:
2390                 strcat(expr, "-");
2391                 break;
2392         case FIELD_OP_PLUS:
2393                 strcat(expr, "+");
2394                 break;
2395         default:
2396                 kfree(expr);
2397                 return NULL;
2398         }
2399
2400         expr_field_str(field->operands[1], expr);
2401
2402         return expr;
2403 }
2404
2405 static int contains_operator(char *str)
2406 {
2407         enum field_op_id field_op = FIELD_OP_NONE;
2408         char *op;
2409
2410         op = strpbrk(str, "+-");
2411         if (!op)
2412                 return FIELD_OP_NONE;
2413
2414         switch (*op) {
2415         case '-':
2416                 if (*str == '-')
2417                         field_op = FIELD_OP_UNARY_MINUS;
2418                 else
2419                         field_op = FIELD_OP_MINUS;
2420                 break;
2421         case '+':
2422                 field_op = FIELD_OP_PLUS;
2423                 break;
2424         default:
2425                 break;
2426         }
2427
2428         return field_op;
2429 }
2430
2431 static void __destroy_hist_field(struct hist_field *hist_field)
2432 {
2433         kfree(hist_field->var.name);
2434         kfree(hist_field->name);
2435         kfree(hist_field->type);
2436
2437         kfree(hist_field);
2438 }
2439
2440 static void destroy_hist_field(struct hist_field *hist_field,
2441                                unsigned int level)
2442 {
2443         unsigned int i;
2444
2445         if (level > 3)
2446                 return;
2447
2448         if (!hist_field)
2449                 return;
2450
2451         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2452                 return; /* var refs will be destroyed separately */
2453
2454         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2455                 destroy_hist_field(hist_field->operands[i], level + 1);
2456
2457         __destroy_hist_field(hist_field);
2458 }
2459
2460 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2461                                             struct ftrace_event_field *field,
2462                                             unsigned long flags,
2463                                             char *var_name)
2464 {
2465         struct hist_field *hist_field;
2466
2467         if (field && is_function_field(field))
2468                 return NULL;
2469
2470         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2471         if (!hist_field)
2472                 return NULL;
2473
2474         hist_field->hist_data = hist_data;
2475
2476         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2477                 goto out; /* caller will populate */
2478
2479         if (flags & HIST_FIELD_FL_VAR_REF) {
2480                 hist_field->fn = hist_field_var_ref;
2481                 goto out;
2482         }
2483
2484         if (flags & HIST_FIELD_FL_HITCOUNT) {
2485                 hist_field->fn = hist_field_counter;
2486                 hist_field->size = sizeof(u64);
2487                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2488                 if (!hist_field->type)
2489                         goto free;
2490                 goto out;
2491         }
2492
2493         if (flags & HIST_FIELD_FL_STACKTRACE) {
2494                 hist_field->fn = hist_field_none;
2495                 goto out;
2496         }
2497
2498         if (flags & HIST_FIELD_FL_LOG2) {
2499                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2500                 hist_field->fn = hist_field_log2;
2501                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2502                 hist_field->size = hist_field->operands[0]->size;
2503                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2504                 if (!hist_field->type)
2505                         goto free;
2506                 goto out;
2507         }
2508
2509         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2510                 hist_field->fn = hist_field_timestamp;
2511                 hist_field->size = sizeof(u64);
2512                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2513                 if (!hist_field->type)
2514                         goto free;
2515                 goto out;
2516         }
2517
2518         if (flags & HIST_FIELD_FL_CPU) {
2519                 hist_field->fn = hist_field_cpu;
2520                 hist_field->size = sizeof(int);
2521                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2522                 if (!hist_field->type)
2523                         goto free;
2524                 goto out;
2525         }
2526
2527         if (WARN_ON_ONCE(!field))
2528                 goto out;
2529
2530         if (is_string_field(field)) {
2531                 flags |= HIST_FIELD_FL_STRING;
2532
2533                 hist_field->size = MAX_FILTER_STR_VAL;
2534                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2535                 if (!hist_field->type)
2536                         goto free;
2537
2538                 if (field->filter_type == FILTER_STATIC_STRING)
2539                         hist_field->fn = hist_field_string;
2540                 else if (field->filter_type == FILTER_DYN_STRING)
2541                         hist_field->fn = hist_field_dynstring;
2542                 else
2543                         hist_field->fn = hist_field_pstring;
2544         } else {
2545                 hist_field->size = field->size;
2546                 hist_field->is_signed = field->is_signed;
2547                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2548                 if (!hist_field->type)
2549                         goto free;
2550
2551                 hist_field->fn = select_value_fn(field->size,
2552                                                  field->is_signed);
2553                 if (!hist_field->fn) {
2554                         destroy_hist_field(hist_field, 0);
2555                         return NULL;
2556                 }
2557         }
2558  out:
2559         hist_field->field = field;
2560         hist_field->flags = flags;
2561
2562         if (var_name) {
2563                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2564                 if (!hist_field->var.name)
2565                         goto free;
2566         }
2567
2568         return hist_field;
2569  free:
2570         destroy_hist_field(hist_field, 0);
2571         return NULL;
2572 }
2573
2574 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2575 {
2576         unsigned int i;
2577
2578         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2579                 if (hist_data->fields[i]) {
2580                         destroy_hist_field(hist_data->fields[i], 0);
2581                         hist_data->fields[i] = NULL;
2582                 }
2583         }
2584
2585         for (i = 0; i < hist_data->n_var_refs; i++) {
2586                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2587                 __destroy_hist_field(hist_data->var_refs[i]);
2588                 hist_data->var_refs[i] = NULL;
2589         }
2590 }
2591
2592 static int init_var_ref(struct hist_field *ref_field,
2593                         struct hist_field *var_field,
2594                         char *system, char *event_name)
2595 {
2596         int err = 0;
2597
2598         ref_field->var.idx = var_field->var.idx;
2599         ref_field->var.hist_data = var_field->hist_data;
2600         ref_field->size = var_field->size;
2601         ref_field->is_signed = var_field->is_signed;
2602         ref_field->flags |= var_field->flags &
2603                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2604
2605         if (system) {
2606                 ref_field->system = kstrdup(system, GFP_KERNEL);
2607                 if (!ref_field->system)
2608                         return -ENOMEM;
2609         }
2610
2611         if (event_name) {
2612                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2613                 if (!ref_field->event_name) {
2614                         err = -ENOMEM;
2615                         goto free;
2616                 }
2617         }
2618
2619         if (var_field->var.name) {
2620                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2621                 if (!ref_field->name) {
2622                         err = -ENOMEM;
2623                         goto free;
2624                 }
2625         } else if (var_field->name) {
2626                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2627                 if (!ref_field->name) {
2628                         err = -ENOMEM;
2629                         goto free;
2630                 }
2631         }
2632
2633         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2634         if (!ref_field->type) {
2635                 err = -ENOMEM;
2636                 goto free;
2637         }
2638  out:
2639         return err;
2640  free:
2641         kfree(ref_field->system);
2642         kfree(ref_field->event_name);
2643         kfree(ref_field->name);
2644
2645         goto out;
2646 }
2647
2648 /**
2649  * create_var_ref - Create a variable reference and attach it to trigger
2650  * @hist_data: The trigger that will be referencing the variable
2651  * @var_field: The VAR field to create a reference to
2652  * @system: The optional system string
2653  * @event_name: The optional event_name string
2654  *
2655  * Given a variable hist_field, create a VAR_REF hist_field that
2656  * represents a reference to it.
2657  *
2658  * This function also adds the reference to the trigger that
2659  * now references the variable.
2660  *
2661  * Return: The VAR_REF field if successful, NULL if not
2662  */
2663 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2664                                          struct hist_field *var_field,
2665                                          char *system, char *event_name)
2666 {
2667         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2668         struct hist_field *ref_field;
2669
2670         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2671         if (ref_field) {
2672                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2673                         destroy_hist_field(ref_field, 0);
2674                         return NULL;
2675                 }
2676
2677                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2678                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2679         }
2680
2681         return ref_field;
2682 }
2683
2684 static bool is_var_ref(char *var_name)
2685 {
2686         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2687                 return false;
2688
2689         return true;
2690 }
2691
2692 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2693                                  char *var_name)
2694 {
2695         char *name, *field;
2696         unsigned int i;
2697
2698         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2699                 name = hist_data->attrs->var_defs.name[i];
2700
2701                 if (strcmp(var_name, name) == 0) {
2702                         field = hist_data->attrs->var_defs.expr[i];
2703                         if (contains_operator(field) || is_var_ref(field))
2704                                 continue;
2705                         return field;
2706                 }
2707         }
2708
2709         return NULL;
2710 }
2711
2712 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2713                                  char *system, char *event_name,
2714                                  char *var_name)
2715 {
2716         struct trace_event_call *call;
2717
2718         if (system && event_name) {
2719                 call = hist_data->event_file->event_call;
2720
2721                 if (strcmp(system, call->class->system) != 0)
2722                         return NULL;
2723
2724                 if (strcmp(event_name, trace_event_name(call)) != 0)
2725                         return NULL;
2726         }
2727
2728         if (!!system != !!event_name)
2729                 return NULL;
2730
2731         if (!is_var_ref(var_name))
2732                 return NULL;
2733
2734         var_name++;
2735
2736         return field_name_from_var(hist_data, var_name);
2737 }
2738
2739 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2740                                         char *system, char *event_name,
2741                                         char *var_name)
2742 {
2743         struct hist_field *var_field = NULL, *ref_field = NULL;
2744         struct trace_array *tr = hist_data->event_file->tr;
2745
2746         if (!is_var_ref(var_name))
2747                 return NULL;
2748
2749         var_name++;
2750
2751         var_field = find_event_var(hist_data, system, event_name, var_name);
2752         if (var_field)
2753                 ref_field = create_var_ref(hist_data, var_field,
2754                                            system, event_name);
2755
2756         if (!ref_field)
2757                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2758
2759         return ref_field;
2760 }
2761
2762 static struct ftrace_event_field *
2763 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2764             char *field_str, unsigned long *flags)
2765 {
2766         struct ftrace_event_field *field = NULL;
2767         char *field_name, *modifier, *str;
2768         struct trace_array *tr = file->tr;
2769
2770         modifier = str = kstrdup(field_str, GFP_KERNEL);
2771         if (!modifier)
2772                 return ERR_PTR(-ENOMEM);
2773
2774         field_name = strsep(&modifier, ".");
2775         if (modifier) {
2776                 if (strcmp(modifier, "hex") == 0)
2777                         *flags |= HIST_FIELD_FL_HEX;
2778                 else if (strcmp(modifier, "sym") == 0)
2779                         *flags |= HIST_FIELD_FL_SYM;
2780                 else if (strcmp(modifier, "sym-offset") == 0)
2781                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2782                 else if ((strcmp(modifier, "execname") == 0) &&
2783                          (strcmp(field_name, "common_pid") == 0))
2784                         *flags |= HIST_FIELD_FL_EXECNAME;
2785                 else if (strcmp(modifier, "syscall") == 0)
2786                         *flags |= HIST_FIELD_FL_SYSCALL;
2787                 else if (strcmp(modifier, "log2") == 0)
2788                         *flags |= HIST_FIELD_FL_LOG2;
2789                 else if (strcmp(modifier, "usecs") == 0)
2790                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2791                 else {
2792                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2793                         field = ERR_PTR(-EINVAL);
2794                         goto out;
2795                 }
2796         }
2797
2798         if (strcmp(field_name, "common_timestamp") == 0) {
2799                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2800                 hist_data->enable_timestamps = true;
2801                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2802                         hist_data->attrs->ts_in_usecs = true;
2803         } else if (strcmp(field_name, "cpu") == 0)
2804                 *flags |= HIST_FIELD_FL_CPU;
2805         else {
2806                 field = trace_find_event_field(file->event_call, field_name);
2807                 if (!field || !field->size) {
2808                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
2809                         field = ERR_PTR(-EINVAL);
2810                         goto out;
2811                 }
2812         }
2813  out:
2814         kfree(str);
2815
2816         return field;
2817 }
2818
2819 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2820                                        struct hist_field *var_ref,
2821                                        char *var_name)
2822 {
2823         struct hist_field *alias = NULL;
2824         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2825
2826         alias = create_hist_field(hist_data, NULL, flags, var_name);
2827         if (!alias)
2828                 return NULL;
2829
2830         alias->fn = var_ref->fn;
2831         alias->operands[0] = var_ref;
2832
2833         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2834                 destroy_hist_field(alias, 0);
2835                 return NULL;
2836         }
2837
2838         alias->var_ref_idx = var_ref->var_ref_idx;
2839
2840         return alias;
2841 }
2842
2843 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2844                                      struct trace_event_file *file, char *str,
2845                                      unsigned long *flags, char *var_name)
2846 {
2847         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2848         struct ftrace_event_field *field = NULL;
2849         struct hist_field *hist_field = NULL;
2850         int ret = 0;
2851
2852         s = strchr(str, '.');
2853         if (s) {
2854                 s = strchr(++s, '.');
2855                 if (s) {
2856                         ref_system = strsep(&str, ".");
2857                         if (!str) {
2858                                 ret = -EINVAL;
2859                                 goto out;
2860                         }
2861                         ref_event = strsep(&str, ".");
2862                         if (!str) {
2863                                 ret = -EINVAL;
2864                                 goto out;
2865                         }
2866                         ref_var = str;
2867                 }
2868         }
2869
2870         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2871         if (!s) {
2872                 hist_field = parse_var_ref(hist_data, ref_system,
2873                                            ref_event, ref_var);
2874                 if (hist_field) {
2875                         if (var_name) {
2876                                 hist_field = create_alias(hist_data, hist_field, var_name);
2877                                 if (!hist_field) {
2878                                         ret = -ENOMEM;
2879                                         goto out;
2880                                 }
2881                         }
2882                         return hist_field;
2883                 }
2884         } else
2885                 str = s;
2886
2887         field = parse_field(hist_data, file, str, flags);
2888         if (IS_ERR(field)) {
2889                 ret = PTR_ERR(field);
2890                 goto out;
2891         }
2892
2893         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2894         if (!hist_field) {
2895                 ret = -ENOMEM;
2896                 goto out;
2897         }
2898
2899         return hist_field;
2900  out:
2901         return ERR_PTR(ret);
2902 }
2903
2904 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2905                                      struct trace_event_file *file,
2906                                      char *str, unsigned long flags,
2907                                      char *var_name, unsigned int level);
2908
2909 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2910                                       struct trace_event_file *file,
2911                                       char *str, unsigned long flags,
2912                                       char *var_name, unsigned int level)
2913 {
2914         struct hist_field *operand1, *expr = NULL;
2915         unsigned long operand_flags;
2916         int ret = 0;
2917         char *s;
2918
2919         /* we support only -(xxx) i.e. explicit parens required */
2920
2921         if (level > 3) {
2922                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2923                 ret = -EINVAL;
2924                 goto free;
2925         }
2926
2927         str++; /* skip leading '-' */
2928
2929         s = strchr(str, '(');
2930         if (s)
2931                 str++;
2932         else {
2933                 ret = -EINVAL;
2934                 goto free;
2935         }
2936
2937         s = strrchr(str, ')');
2938         if (s)
2939                 *s = '\0';
2940         else {
2941                 ret = -EINVAL; /* no closing ')' */
2942                 goto free;
2943         }
2944
2945         flags |= HIST_FIELD_FL_EXPR;
2946         expr = create_hist_field(hist_data, NULL, flags, var_name);
2947         if (!expr) {
2948                 ret = -ENOMEM;
2949                 goto free;
2950         }
2951
2952         operand_flags = 0;
2953         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2954         if (IS_ERR(operand1)) {
2955                 ret = PTR_ERR(operand1);
2956                 goto free;
2957         }
2958
2959         expr->flags |= operand1->flags &
2960                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2961         expr->fn = hist_field_unary_minus;
2962         expr->operands[0] = operand1;
2963         expr->operator = FIELD_OP_UNARY_MINUS;
2964         expr->name = expr_str(expr, 0);
2965         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2966         if (!expr->type) {
2967                 ret = -ENOMEM;
2968                 goto free;
2969         }
2970
2971         return expr;
2972  free:
2973         destroy_hist_field(expr, 0);
2974         return ERR_PTR(ret);
2975 }
2976
2977 static int check_expr_operands(struct trace_array *tr,
2978                                struct hist_field *operand1,
2979                                struct hist_field *operand2)
2980 {
2981         unsigned long operand1_flags = operand1->flags;
2982         unsigned long operand2_flags = operand2->flags;
2983
2984         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2985             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2986                 struct hist_field *var;
2987
2988                 var = find_var_field(operand1->var.hist_data, operand1->name);
2989                 if (!var)
2990                         return -EINVAL;
2991                 operand1_flags = var->flags;
2992         }
2993
2994         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2995             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2996                 struct hist_field *var;
2997
2998                 var = find_var_field(operand2->var.hist_data, operand2->name);
2999                 if (!var)
3000                         return -EINVAL;
3001                 operand2_flags = var->flags;
3002         }
3003
3004         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
3005             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
3006                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
3007                 return -EINVAL;
3008         }
3009
3010         return 0;
3011 }
3012
3013 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
3014                                      struct trace_event_file *file,
3015                                      char *str, unsigned long flags,
3016                                      char *var_name, unsigned int level)
3017 {
3018         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
3019         unsigned long operand_flags;
3020         int field_op, ret = -EINVAL;
3021         char *sep, *operand1_str;
3022
3023         if (level > 3) {
3024                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
3025                 return ERR_PTR(-EINVAL);
3026         }
3027
3028         field_op = contains_operator(str);
3029
3030         if (field_op == FIELD_OP_NONE)
3031                 return parse_atom(hist_data, file, str, &flags, var_name);
3032
3033         if (field_op == FIELD_OP_UNARY_MINUS)
3034                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
3035
3036         switch (field_op) {
3037         case FIELD_OP_MINUS:
3038                 sep = "-";
3039                 break;
3040         case FIELD_OP_PLUS:
3041                 sep = "+";
3042                 break;
3043         default:
3044                 goto free;
3045         }
3046
3047         operand1_str = strsep(&str, sep);
3048         if (!operand1_str || !str)
3049                 goto free;
3050
3051         operand_flags = 0;
3052         operand1 = parse_atom(hist_data, file, operand1_str,
3053                               &operand_flags, NULL);
3054         if (IS_ERR(operand1)) {
3055                 ret = PTR_ERR(operand1);
3056                 operand1 = NULL;
3057                 goto free;
3058         }
3059
3060         /* rest of string could be another expression e.g. b+c in a+b+c */
3061         operand_flags = 0;
3062         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3063         if (IS_ERR(operand2)) {
3064                 ret = PTR_ERR(operand2);
3065                 operand2 = NULL;
3066                 goto free;
3067         }
3068
3069         ret = check_expr_operands(file->tr, operand1, operand2);
3070         if (ret)
3071                 goto free;
3072
3073         flags |= HIST_FIELD_FL_EXPR;
3074
3075         flags |= operand1->flags &
3076                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3077
3078         expr = create_hist_field(hist_data, NULL, flags, var_name);
3079         if (!expr) {
3080                 ret = -ENOMEM;
3081                 goto free;
3082         }
3083
3084         operand1->read_once = true;
3085         operand2->read_once = true;
3086
3087         expr->operands[0] = operand1;
3088         expr->operands[1] = operand2;
3089         expr->operator = field_op;
3090         expr->name = expr_str(expr, 0);
3091         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3092         if (!expr->type) {
3093                 ret = -ENOMEM;
3094                 goto free;
3095         }
3096
3097         switch (field_op) {
3098         case FIELD_OP_MINUS:
3099                 expr->fn = hist_field_minus;
3100                 break;
3101         case FIELD_OP_PLUS:
3102                 expr->fn = hist_field_plus;
3103                 break;
3104         default:
3105                 ret = -EINVAL;
3106                 goto free;
3107         }
3108
3109         return expr;
3110  free:
3111         destroy_hist_field(operand1, 0);
3112         destroy_hist_field(operand2, 0);
3113         destroy_hist_field(expr, 0);
3114
3115         return ERR_PTR(ret);
3116 }
3117
3118 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3119                                  struct trace_event_file *file)
3120 {
3121         struct event_trigger_data *test;
3122
3123         lockdep_assert_held(&event_mutex);
3124
3125         list_for_each_entry(test, &file->triggers, list) {
3126                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3127                         if (test->private_data == hist_data)
3128                                 return test->filter_str;
3129                 }
3130         }
3131
3132         return NULL;
3133 }
3134
3135 static struct event_command trigger_hist_cmd;
3136 static int event_hist_trigger_func(struct event_command *cmd_ops,
3137                                    struct trace_event_file *file,
3138                                    char *glob, char *cmd, char *param);
3139
3140 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3141                             struct hist_trigger_data *hist_data,
3142                             unsigned int n_keys)
3143 {
3144         struct hist_field *target_hist_field, *hist_field;
3145         unsigned int n, i, j;
3146
3147         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3148                 return false;
3149
3150         i = hist_data->n_vals;
3151         j = target_hist_data->n_vals;
3152
3153         for (n = 0; n < n_keys; n++) {
3154                 hist_field = hist_data->fields[i + n];
3155                 target_hist_field = target_hist_data->fields[j + n];
3156
3157                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3158                         return false;
3159                 if (hist_field->size != target_hist_field->size)
3160                         return false;
3161                 if (hist_field->is_signed != target_hist_field->is_signed)
3162                         return false;
3163         }
3164
3165         return true;
3166 }
3167
3168 static struct hist_trigger_data *
3169 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3170                      struct trace_event_file *file)
3171 {
3172         struct hist_trigger_data *hist_data;
3173         struct event_trigger_data *test;
3174         unsigned int n_keys;
3175
3176         lockdep_assert_held(&event_mutex);
3177
3178         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3179
3180         list_for_each_entry(test, &file->triggers, list) {
3181                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3182                         hist_data = test->private_data;
3183
3184                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3185                                 return hist_data;
3186                 }
3187         }
3188
3189         return NULL;
3190 }
3191
3192 static struct trace_event_file *event_file(struct trace_array *tr,
3193                                            char *system, char *event_name)
3194 {
3195         struct trace_event_file *file;
3196
3197         file = __find_event_file(tr, system, event_name);
3198         if (!file)
3199                 return ERR_PTR(-EINVAL);
3200
3201         return file;
3202 }
3203
3204 static struct hist_field *
3205 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3206                          char *system, char *event_name, char *field_name)
3207 {
3208         struct hist_field *event_var;
3209         char *synthetic_name;
3210
3211         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3212         if (!synthetic_name)
3213                 return ERR_PTR(-ENOMEM);
3214
3215         strcpy(synthetic_name, "synthetic_");
3216         strcat(synthetic_name, field_name);
3217
3218         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3219
3220         kfree(synthetic_name);
3221
3222         return event_var;
3223 }
3224
3225 /**
3226  * create_field_var_hist - Automatically create a histogram and var for a field
3227  * @target_hist_data: The target hist trigger
3228  * @subsys_name: Optional subsystem name
3229  * @event_name: Optional event name
3230  * @field_name: The name of the field (and the resulting variable)
3231  *
3232  * Hist trigger actions fetch data from variables, not directly from
3233  * events.  However, for convenience, users are allowed to directly
3234  * specify an event field in an action, which will be automatically
3235  * converted into a variable on their behalf.
3236
3237  * If a user specifies a field on an event that isn't the event the
3238  * histogram currently being defined (the target event histogram), the
3239  * only way that can be accomplished is if a new hist trigger is
3240  * created and the field variable defined on that.
3241  *
3242  * This function creates a new histogram compatible with the target
3243  * event (meaning a histogram with the same key as the target
3244  * histogram), and creates a variable for the specified field, but
3245  * with 'synthetic_' prepended to the variable name in order to avoid
3246  * collision with normal field variables.
3247  *
3248  * Return: The variable created for the field.
3249  */
3250 static struct hist_field *
3251 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3252                       char *subsys_name, char *event_name, char *field_name)
3253 {
3254         struct trace_array *tr = target_hist_data->event_file->tr;
3255         struct hist_field *event_var = ERR_PTR(-EINVAL);
3256         struct hist_trigger_data *hist_data;
3257         unsigned int i, n, first = true;
3258         struct field_var_hist *var_hist;
3259         struct trace_event_file *file;
3260         struct hist_field *key_field;
3261         char *saved_filter;
3262         char *cmd;
3263         int ret;
3264
3265         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3266                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3267                 return ERR_PTR(-EINVAL);
3268         }
3269
3270         file = event_file(tr, subsys_name, event_name);
3271
3272         if (IS_ERR(file)) {
3273                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3274                 ret = PTR_ERR(file);
3275                 return ERR_PTR(ret);
3276         }
3277
3278         /*
3279          * Look for a histogram compatible with target.  We'll use the
3280          * found histogram specification to create a new matching
3281          * histogram with our variable on it.  target_hist_data is not
3282          * yet a registered histogram so we can't use that.
3283          */
3284         hist_data = find_compatible_hist(target_hist_data, file);
3285         if (!hist_data) {
3286                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3287                 return ERR_PTR(-EINVAL);
3288         }
3289
3290         /* See if a synthetic field variable has already been created */
3291         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3292                                              event_name, field_name);
3293         if (!IS_ERR_OR_NULL(event_var))
3294                 return event_var;
3295
3296         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3297         if (!var_hist)
3298                 return ERR_PTR(-ENOMEM);
3299
3300         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3301         if (!cmd) {
3302                 kfree(var_hist);
3303                 return ERR_PTR(-ENOMEM);
3304         }
3305
3306         /* Use the same keys as the compatible histogram */
3307         strcat(cmd, "keys=");
3308
3309         for_each_hist_key_field(i, hist_data) {
3310                 key_field = hist_data->fields[i];
3311                 if (!first)
3312                         strcat(cmd, ",");
3313                 strcat(cmd, key_field->field->name);
3314                 first = false;
3315         }
3316
3317         /* Create the synthetic field variable specification */
3318         strcat(cmd, ":synthetic_");
3319         strcat(cmd, field_name);
3320         strcat(cmd, "=");
3321         strcat(cmd, field_name);
3322
3323         /* Use the same filter as the compatible histogram */
3324         saved_filter = find_trigger_filter(hist_data, file);
3325         if (saved_filter) {
3326                 strcat(cmd, " if ");
3327                 strcat(cmd, saved_filter);
3328         }
3329
3330         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3331         if (!var_hist->cmd) {
3332                 kfree(cmd);
3333                 kfree(var_hist);
3334                 return ERR_PTR(-ENOMEM);
3335         }
3336
3337         /* Save the compatible histogram information */
3338         var_hist->hist_data = hist_data;
3339
3340         /* Create the new histogram with our variable */
3341         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3342                                       "", "hist", cmd);
3343         if (ret) {
3344                 kfree(cmd);
3345                 kfree(var_hist->cmd);
3346                 kfree(var_hist);
3347                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3348                 return ERR_PTR(ret);
3349         }
3350
3351         kfree(cmd);
3352
3353         /* If we can't find the variable, something went wrong */
3354         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3355                                              event_name, field_name);
3356         if (IS_ERR_OR_NULL(event_var)) {
3357                 kfree(var_hist->cmd);
3358                 kfree(var_hist);
3359                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3360                 return ERR_PTR(-EINVAL);
3361         }
3362
3363         n = target_hist_data->n_field_var_hists;
3364         target_hist_data->field_var_hists[n] = var_hist;
3365         target_hist_data->n_field_var_hists++;
3366
3367         return event_var;
3368 }
3369
3370 static struct hist_field *
3371 find_target_event_var(struct hist_trigger_data *hist_data,
3372                       char *subsys_name, char *event_name, char *var_name)
3373 {
3374         struct trace_event_file *file = hist_data->event_file;
3375         struct hist_field *hist_field = NULL;
3376
3377         if (subsys_name) {
3378                 struct trace_event_call *call;
3379
3380                 if (!event_name)
3381                         return NULL;
3382
3383                 call = file->event_call;
3384
3385                 if (strcmp(subsys_name, call->class->system) != 0)
3386                         return NULL;
3387
3388                 if (strcmp(event_name, trace_event_name(call)) != 0)
3389                         return NULL;
3390         }
3391
3392         hist_field = find_var_field(hist_data, var_name);
3393
3394         return hist_field;
3395 }
3396
3397 static inline void __update_field_vars(struct tracing_map_elt *elt,
3398                                        struct ring_buffer_event *rbe,
3399                                        void *rec,
3400                                        struct field_var **field_vars,
3401                                        unsigned int n_field_vars,
3402                                        unsigned int field_var_str_start)
3403 {
3404         struct hist_elt_data *elt_data = elt->private_data;
3405         unsigned int i, j, var_idx;
3406         u64 var_val;
3407
3408         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3409                 struct field_var *field_var = field_vars[i];
3410                 struct hist_field *var = field_var->var;
3411                 struct hist_field *val = field_var->val;
3412
3413                 var_val = val->fn(val, elt, rbe, rec);
3414                 var_idx = var->var.idx;
3415
3416                 if (val->flags & HIST_FIELD_FL_STRING) {
3417                         char *str = elt_data->field_var_str[j++];
3418                         char *val_str = (char *)(uintptr_t)var_val;
3419
3420                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3421                         var_val = (u64)(uintptr_t)str;
3422                 }
3423                 tracing_map_set_var(elt, var_idx, var_val);
3424         }
3425 }
3426
3427 static void update_field_vars(struct hist_trigger_data *hist_data,
3428                               struct tracing_map_elt *elt,
3429                               struct ring_buffer_event *rbe,
3430                               void *rec)
3431 {
3432         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3433                             hist_data->n_field_vars, 0);
3434 }
3435
3436 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3437                                  struct tracing_map_elt *elt, void *rec,
3438                                  struct ring_buffer_event *rbe, void *key,
3439                                  struct action_data *data, u64 *var_ref_vals)
3440 {
3441         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
3442                             hist_data->n_save_vars, hist_data->n_field_var_str);
3443 }
3444
3445 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3446                                      struct trace_event_file *file,
3447                                      char *name, int size, const char *type)
3448 {
3449         struct hist_field *var;
3450         int idx;
3451
3452         if (find_var(hist_data, file, name) && !hist_data->remove) {
3453                 var = ERR_PTR(-EINVAL);
3454                 goto out;
3455         }
3456
3457         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3458         if (!var) {
3459                 var = ERR_PTR(-ENOMEM);
3460                 goto out;
3461         }
3462
3463         idx = tracing_map_add_var(hist_data->map);
3464         if (idx < 0) {
3465                 kfree(var);
3466                 var = ERR_PTR(-EINVAL);
3467                 goto out;
3468         }
3469
3470         var->flags = HIST_FIELD_FL_VAR;
3471         var->var.idx = idx;
3472         var->var.hist_data = var->hist_data = hist_data;
3473         var->size = size;
3474         var->var.name = kstrdup(name, GFP_KERNEL);
3475         var->type = kstrdup(type, GFP_KERNEL);
3476         if (!var->var.name || !var->type) {
3477                 kfree(var->var.name);
3478                 kfree(var->type);
3479                 kfree(var);
3480                 var = ERR_PTR(-ENOMEM);
3481         }
3482  out:
3483         return var;
3484 }
3485
3486 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3487                                           struct trace_event_file *file,
3488                                           char *field_name)
3489 {
3490         struct hist_field *val = NULL, *var = NULL;
3491         unsigned long flags = HIST_FIELD_FL_VAR;
3492         struct trace_array *tr = file->tr;
3493         struct field_var *field_var;
3494         int ret = 0;
3495
3496         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3497                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3498                 ret = -EINVAL;
3499                 goto err;
3500         }
3501
3502         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3503         if (IS_ERR(val)) {
3504                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3505                 ret = PTR_ERR(val);
3506                 goto err;
3507         }
3508
3509         var = create_var(hist_data, file, field_name, val->size, val->type);
3510         if (IS_ERR(var)) {
3511                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3512                 kfree(val);
3513                 ret = PTR_ERR(var);
3514                 goto err;
3515         }
3516
3517         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3518         if (!field_var) {
3519                 kfree(val);
3520                 kfree(var);
3521                 ret =  -ENOMEM;
3522                 goto err;
3523         }
3524
3525         field_var->var = var;
3526         field_var->val = val;
3527  out:
3528         return field_var;
3529  err:
3530         field_var = ERR_PTR(ret);
3531         goto out;
3532 }
3533
3534 /**
3535  * create_target_field_var - Automatically create a variable for a field
3536  * @target_hist_data: The target hist trigger
3537  * @subsys_name: Optional subsystem name
3538  * @event_name: Optional event name
3539  * @var_name: The name of the field (and the resulting variable)
3540  *
3541  * Hist trigger actions fetch data from variables, not directly from
3542  * events.  However, for convenience, users are allowed to directly
3543  * specify an event field in an action, which will be automatically
3544  * converted into a variable on their behalf.
3545
3546  * This function creates a field variable with the name var_name on
3547  * the hist trigger currently being defined on the target event.  If
3548  * subsys_name and event_name are specified, this function simply
3549  * verifies that they do in fact match the target event subsystem and
3550  * event name.
3551  *
3552  * Return: The variable created for the field.
3553  */
3554 static struct field_var *
3555 create_target_field_var(struct hist_trigger_data *target_hist_data,
3556                         char *subsys_name, char *event_name, char *var_name)
3557 {
3558         struct trace_event_file *file = target_hist_data->event_file;
3559
3560         if (subsys_name) {
3561                 struct trace_event_call *call;
3562
3563                 if (!event_name)
3564                         return NULL;
3565
3566                 call = file->event_call;
3567
3568                 if (strcmp(subsys_name, call->class->system) != 0)
3569                         return NULL;
3570
3571                 if (strcmp(event_name, trace_event_name(call)) != 0)
3572                         return NULL;
3573         }
3574
3575         return create_field_var(target_hist_data, file, var_name);
3576 }
3577
3578 static bool check_track_val_max(u64 track_val, u64 var_val)
3579 {
3580         if (var_val <= track_val)
3581                 return false;
3582
3583         return true;
3584 }
3585
3586 static bool check_track_val_changed(u64 track_val, u64 var_val)
3587 {
3588         if (var_val == track_val)
3589                 return false;
3590
3591         return true;
3592 }
3593
3594 static u64 get_track_val(struct hist_trigger_data *hist_data,
3595                          struct tracing_map_elt *elt,
3596                          struct action_data *data)
3597 {
3598         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3599         u64 track_val;
3600
3601         track_val = tracing_map_read_var(elt, track_var_idx);
3602
3603         return track_val;
3604 }
3605
3606 static void save_track_val(struct hist_trigger_data *hist_data,
3607                            struct tracing_map_elt *elt,
3608                            struct action_data *data, u64 var_val)
3609 {
3610         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3611
3612         tracing_map_set_var(elt, track_var_idx, var_val);
3613 }
3614
3615 static void save_track_data(struct hist_trigger_data *hist_data,
3616                             struct tracing_map_elt *elt, void *rec,
3617                             struct ring_buffer_event *rbe, void *key,
3618                             struct action_data *data, u64 *var_ref_vals)
3619 {
3620         if (data->track_data.save_data)
3621                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3622 }
3623
3624 static bool check_track_val(struct tracing_map_elt *elt,
3625                             struct action_data *data,
3626                             u64 var_val)
3627 {
3628         struct hist_trigger_data *hist_data;
3629         u64 track_val;
3630
3631         hist_data = data->track_data.track_var->hist_data;
3632         track_val = get_track_val(hist_data, elt, data);
3633
3634         return data->track_data.check_val(track_val, var_val);
3635 }
3636
3637 #ifdef CONFIG_TRACER_SNAPSHOT
3638 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3639 {
3640         /* called with tr->max_lock held */
3641         struct track_data *track_data = tr->cond_snapshot->cond_data;
3642         struct hist_elt_data *elt_data, *track_elt_data;
3643         struct snapshot_context *context = cond_data;
3644         struct action_data *action;
3645         u64 track_val;
3646
3647         if (!track_data)
3648                 return false;
3649
3650         action = track_data->action_data;
3651
3652         track_val = get_track_val(track_data->hist_data, context->elt,
3653                                   track_data->action_data);
3654
3655         if (!action->track_data.check_val(track_data->track_val, track_val))
3656                 return false;
3657
3658         track_data->track_val = track_val;
3659         memcpy(track_data->key, context->key, track_data->key_len);
3660
3661         elt_data = context->elt->private_data;
3662         track_elt_data = track_data->elt.private_data;
3663         if (elt_data->comm)
3664                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3665
3666         track_data->updated = true;
3667
3668         return true;
3669 }
3670
3671 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3672                                      struct tracing_map_elt *elt, void *rec,
3673                                      struct ring_buffer_event *rbe, void *key,
3674                                      struct action_data *data,
3675                                      u64 *var_ref_vals)
3676 {
3677         struct trace_event_file *file = hist_data->event_file;
3678         struct snapshot_context context;
3679
3680         context.elt = elt;
3681         context.key = key;
3682
3683         tracing_snapshot_cond(file->tr, &context);
3684 }
3685
3686 static void hist_trigger_print_key(struct seq_file *m,
3687                                    struct hist_trigger_data *hist_data,
3688                                    void *key,
3689                                    struct tracing_map_elt *elt);
3690
3691 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3692 {
3693         unsigned int i;
3694
3695         if (!hist_data->n_actions)
3696                 return NULL;
3697
3698         for (i = 0; i < hist_data->n_actions; i++) {
3699                 struct action_data *data = hist_data->actions[i];
3700
3701                 if (data->action == ACTION_SNAPSHOT)
3702                         return data;
3703         }
3704
3705         return NULL;
3706 }
3707
3708 static void track_data_snapshot_print(struct seq_file *m,
3709                                       struct hist_trigger_data *hist_data)
3710 {
3711         struct trace_event_file *file = hist_data->event_file;
3712         struct track_data *track_data;
3713         struct action_data *action;
3714
3715         track_data = tracing_cond_snapshot_data(file->tr);
3716         if (!track_data)
3717                 return;
3718
3719         if (!track_data->updated)
3720                 return;
3721
3722         action = snapshot_action(hist_data);
3723         if (!action)
3724                 return;
3725
3726         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3727         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3728                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3729                    action->track_data.var_str, track_data->track_val);
3730
3731         seq_puts(m, "\ttriggered by event with key: ");
3732         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3733         seq_putc(m, '\n');
3734 }
3735 #else
3736 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3737 {
3738         return false;
3739 }
3740 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3741                                      struct tracing_map_elt *elt, void *rec,
3742                                      struct ring_buffer_event *rbe, void *key,
3743                                      struct action_data *data,
3744                                      u64 *var_ref_vals) {}
3745 static void track_data_snapshot_print(struct seq_file *m,
3746                                       struct hist_trigger_data *hist_data) {}
3747 #endif /* CONFIG_TRACER_SNAPSHOT */
3748
3749 static void track_data_print(struct seq_file *m,
3750                              struct hist_trigger_data *hist_data,
3751                              struct tracing_map_elt *elt,
3752                              struct action_data *data)
3753 {
3754         u64 track_val = get_track_val(hist_data, elt, data);
3755         unsigned int i, save_var_idx;
3756
3757         if (data->handler == HANDLER_ONMAX)
3758                 seq_printf(m, "\n\tmax: %10llu", track_val);
3759         else if (data->handler == HANDLER_ONCHANGE)
3760                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3761
3762         if (data->action == ACTION_SNAPSHOT)
3763                 return;
3764
3765         for (i = 0; i < hist_data->n_save_vars; i++) {
3766                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3767                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3768                 u64 val;
3769
3770                 save_var_idx = save_var->var.idx;
3771
3772                 val = tracing_map_read_var(elt, save_var_idx);
3773
3774                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3775                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3776                                    (char *)(uintptr_t)(val));
3777                 } else
3778                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3779         }
3780 }
3781
3782 static void ontrack_action(struct hist_trigger_data *hist_data,
3783                            struct tracing_map_elt *elt, void *rec,
3784                            struct ring_buffer_event *rbe, void *key,
3785                            struct action_data *data, u64 *var_ref_vals)
3786 {
3787         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3788
3789         if (check_track_val(elt, data, var_val)) {
3790                 save_track_val(hist_data, elt, data, var_val);
3791                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3792         }
3793 }
3794
3795 static void action_data_destroy(struct action_data *data)
3796 {
3797         unsigned int i;
3798
3799         lockdep_assert_held(&event_mutex);
3800
3801         kfree(data->action_name);
3802
3803         for (i = 0; i < data->n_params; i++)
3804                 kfree(data->params[i]);
3805
3806         if (data->synth_event)
3807                 data->synth_event->ref--;
3808
3809         kfree(data->synth_event_name);
3810
3811         kfree(data);
3812 }
3813
3814 static void track_data_destroy(struct hist_trigger_data *hist_data,
3815                                struct action_data *data)
3816 {
3817         struct trace_event_file *file = hist_data->event_file;
3818
3819         destroy_hist_field(data->track_data.track_var, 0);
3820
3821         if (data->action == ACTION_SNAPSHOT) {
3822                 struct track_data *track_data;
3823
3824                 track_data = tracing_cond_snapshot_data(file->tr);
3825                 if (track_data && track_data->hist_data == hist_data) {
3826                         tracing_snapshot_cond_disable(file->tr);
3827                         track_data_free(track_data);
3828                 }
3829         }
3830
3831         kfree(data->track_data.var_str);
3832
3833         action_data_destroy(data);
3834 }
3835
3836 static int action_create(struct hist_trigger_data *hist_data,
3837                          struct action_data *data);
3838
3839 static int track_data_create(struct hist_trigger_data *hist_data,
3840                              struct action_data *data)
3841 {
3842         struct hist_field *var_field, *ref_field, *track_var = NULL;
3843         struct trace_event_file *file = hist_data->event_file;
3844         struct trace_array *tr = file->tr;
3845         char *track_data_var_str;
3846         int ret = 0;
3847
3848         track_data_var_str = data->track_data.var_str;
3849         if (track_data_var_str[0] != '$') {
3850                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3851                 return -EINVAL;
3852         }
3853         track_data_var_str++;
3854
3855         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3856         if (!var_field) {
3857                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3858                 return -EINVAL;
3859         }
3860
3861         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3862         if (!ref_field)
3863                 return -ENOMEM;
3864
3865         data->track_data.var_ref = ref_field;
3866
3867         if (data->handler == HANDLER_ONMAX)
3868                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3869         if (IS_ERR(track_var)) {
3870                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3871                 ret = PTR_ERR(track_var);
3872                 goto out;
3873         }
3874
3875         if (data->handler == HANDLER_ONCHANGE)
3876                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3877         if (IS_ERR(track_var)) {
3878                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3879                 ret = PTR_ERR(track_var);
3880                 goto out;
3881         }
3882         data->track_data.track_var = track_var;
3883
3884         ret = action_create(hist_data, data);
3885  out:
3886         return ret;
3887 }
3888
3889 static int parse_action_params(struct trace_array *tr, char *params,
3890                                struct action_data *data)
3891 {
3892         char *param, *saved_param;
3893         bool first_param = true;
3894         int ret = 0;
3895
3896         while (params) {
3897                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3898                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3899                         goto out;
3900                 }
3901
3902                 param = strsep(&params, ",");
3903                 if (!param) {
3904                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3905                         ret = -EINVAL;
3906                         goto out;
3907                 }
3908
3909                 param = strstrip(param);
3910                 if (strlen(param) < 2) {
3911                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3912                         ret = -EINVAL;
3913                         goto out;
3914                 }
3915
3916                 saved_param = kstrdup(param, GFP_KERNEL);
3917                 if (!saved_param) {
3918                         ret = -ENOMEM;
3919                         goto out;
3920                 }
3921
3922                 if (first_param && data->use_trace_keyword) {
3923                         data->synth_event_name = saved_param;
3924                         first_param = false;
3925                         continue;
3926                 }
3927                 first_param = false;
3928
3929                 data->params[data->n_params++] = saved_param;
3930         }
3931  out:
3932         return ret;
3933 }
3934
3935 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3936                         enum handler_id handler)
3937 {
3938         char *action_name;
3939         int ret = 0;
3940
3941         strsep(&str, ".");
3942         if (!str) {
3943                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3944                 ret = -EINVAL;
3945                 goto out;
3946         }
3947
3948         action_name = strsep(&str, "(");
3949         if (!action_name || !str) {
3950                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3951                 ret = -EINVAL;
3952                 goto out;
3953         }
3954
3955         if (str_has_prefix(action_name, "save")) {
3956                 char *params = strsep(&str, ")");
3957
3958                 if (!params) {
3959                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3960                         ret = -EINVAL;
3961                         goto out;
3962                 }
3963
3964                 ret = parse_action_params(tr, params, data);
3965                 if (ret)
3966                         goto out;
3967
3968                 if (handler == HANDLER_ONMAX)
3969                         data->track_data.check_val = check_track_val_max;
3970                 else if (handler == HANDLER_ONCHANGE)
3971                         data->track_data.check_val = check_track_val_changed;
3972                 else {
3973                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3974                         ret = -EINVAL;
3975                         goto out;
3976                 }
3977
3978                 data->track_data.save_data = save_track_data_vars;
3979                 data->fn = ontrack_action;
3980                 data->action = ACTION_SAVE;
3981         } else if (str_has_prefix(action_name, "snapshot")) {
3982                 char *params = strsep(&str, ")");
3983
3984                 if (!str) {
3985                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3986                         ret = -EINVAL;
3987                         goto out;
3988                 }
3989
3990                 if (handler == HANDLER_ONMAX)
3991                         data->track_data.check_val = check_track_val_max;
3992                 else if (handler == HANDLER_ONCHANGE)
3993                         data->track_data.check_val = check_track_val_changed;
3994                 else {
3995                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3996                         ret = -EINVAL;
3997                         goto out;
3998                 }
3999
4000                 data->track_data.save_data = save_track_data_snapshot;
4001                 data->fn = ontrack_action;
4002                 data->action = ACTION_SNAPSHOT;
4003         } else {
4004                 char *params = strsep(&str, ")");
4005
4006                 if (str_has_prefix(action_name, "trace"))
4007                         data->use_trace_keyword = true;
4008
4009                 if (params) {
4010                         ret = parse_action_params(tr, params, data);
4011                         if (ret)
4012                                 goto out;
4013                 }
4014
4015                 if (handler == HANDLER_ONMAX)
4016                         data->track_data.check_val = check_track_val_max;
4017                 else if (handler == HANDLER_ONCHANGE)
4018                         data->track_data.check_val = check_track_val_changed;
4019
4020                 if (handler != HANDLER_ONMATCH) {
4021                         data->track_data.save_data = action_trace;
4022                         data->fn = ontrack_action;
4023                 } else
4024                         data->fn = action_trace;
4025
4026                 data->action = ACTION_TRACE;
4027         }
4028
4029         data->action_name = kstrdup(action_name, GFP_KERNEL);
4030         if (!data->action_name) {
4031                 ret = -ENOMEM;
4032                 goto out;
4033         }
4034
4035         data->handler = handler;
4036  out:
4037         return ret;
4038 }
4039
4040 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
4041                                             char *str, enum handler_id handler)
4042 {
4043         struct action_data *data;
4044         int ret = -EINVAL;
4045         char *var_str;
4046
4047         data = kzalloc(sizeof(*data), GFP_KERNEL);
4048         if (!data)
4049                 return ERR_PTR(-ENOMEM);
4050
4051         var_str = strsep(&str, ")");
4052         if (!var_str || !str) {
4053                 ret = -EINVAL;
4054                 goto free;
4055         }
4056
4057         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
4058         if (!data->track_data.var_str) {
4059                 ret = -ENOMEM;
4060                 goto free;
4061         }
4062
4063         ret = action_parse(hist_data->event_file->tr, str, data, handler);
4064         if (ret)
4065                 goto free;
4066  out:
4067         return data;
4068  free:
4069         track_data_destroy(hist_data, data);
4070         data = ERR_PTR(ret);
4071         goto out;
4072 }
4073
4074 static void onmatch_destroy(struct action_data *data)
4075 {
4076         kfree(data->match_data.event);
4077         kfree(data->match_data.event_system);
4078
4079         action_data_destroy(data);
4080 }
4081
4082 static void destroy_field_var(struct field_var *field_var)
4083 {
4084         if (!field_var)
4085                 return;
4086
4087         destroy_hist_field(field_var->var, 0);
4088         destroy_hist_field(field_var->val, 0);
4089
4090         kfree(field_var);
4091 }
4092
4093 static void destroy_field_vars(struct hist_trigger_data *hist_data)
4094 {
4095         unsigned int i;
4096
4097         for (i = 0; i < hist_data->n_field_vars; i++)
4098                 destroy_field_var(hist_data->field_vars[i]);
4099 }
4100
4101 static void save_field_var(struct hist_trigger_data *hist_data,
4102                            struct field_var *field_var)
4103 {
4104         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
4105
4106         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4107                 hist_data->n_field_var_str++;
4108 }
4109
4110
4111 static int check_synth_field(struct synth_event *event,
4112                              struct hist_field *hist_field,
4113                              unsigned int field_pos)
4114 {
4115         struct synth_field *field;
4116
4117         if (field_pos >= event->n_fields)
4118                 return -EINVAL;
4119
4120         field = event->fields[field_pos];
4121
4122         if (strcmp(field->type, hist_field->type) != 0) {
4123                 if (field->size != hist_field->size ||
4124                     field->is_signed != hist_field->is_signed)
4125                         return -EINVAL;
4126         }
4127
4128         return 0;
4129 }
4130
4131 static struct hist_field *
4132 trace_action_find_var(struct hist_trigger_data *hist_data,
4133                       struct action_data *data,
4134                       char *system, char *event, char *var)
4135 {
4136         struct trace_array *tr = hist_data->event_file->tr;
4137         struct hist_field *hist_field;
4138
4139         var++; /* skip '$' */
4140
4141         hist_field = find_target_event_var(hist_data, system, event, var);
4142         if (!hist_field) {
4143                 if (!system && data->handler == HANDLER_ONMATCH) {
4144                         system = data->match_data.event_system;
4145                         event = data->match_data.event;
4146                 }
4147
4148                 hist_field = find_event_var(hist_data, system, event, var);
4149         }
4150
4151         if (!hist_field)
4152                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
4153
4154         return hist_field;
4155 }
4156
4157 static struct hist_field *
4158 trace_action_create_field_var(struct hist_trigger_data *hist_data,
4159                               struct action_data *data, char *system,
4160                               char *event, char *var)
4161 {
4162         struct hist_field *hist_field = NULL;
4163         struct field_var *field_var;
4164
4165         /*
4166          * First try to create a field var on the target event (the
4167          * currently being defined).  This will create a variable for
4168          * unqualified fields on the target event, or if qualified,
4169          * target fields that have qualified names matching the target.
4170          */
4171         field_var = create_target_field_var(hist_data, system, event, var);
4172
4173         if (field_var && !IS_ERR(field_var)) {
4174                 save_field_var(hist_data, field_var);
4175                 hist_field = field_var->var;
4176         } else {
4177                 field_var = NULL;
4178                 /*
4179                  * If no explicit system.event is specfied, default to
4180                  * looking for fields on the onmatch(system.event.xxx)
4181                  * event.
4182                  */
4183                 if (!system && data->handler == HANDLER_ONMATCH) {
4184                         system = data->match_data.event_system;
4185                         event = data->match_data.event;
4186                 }
4187
4188                 /*
4189                  * At this point, we're looking at a field on another
4190                  * event.  Because we can't modify a hist trigger on
4191                  * another event to add a variable for a field, we need
4192                  * to create a new trigger on that event and create the
4193                  * variable at the same time.
4194                  */
4195                 hist_field = create_field_var_hist(hist_data, system, event, var);
4196                 if (IS_ERR(hist_field))
4197                         goto free;
4198         }
4199  out:
4200         return hist_field;
4201  free:
4202         destroy_field_var(field_var);
4203         hist_field = NULL;
4204         goto out;
4205 }
4206
4207 static int trace_action_create(struct hist_trigger_data *hist_data,
4208                                struct action_data *data)
4209 {
4210         struct trace_array *tr = hist_data->event_file->tr;
4211         char *event_name, *param, *system = NULL;
4212         struct hist_field *hist_field, *var_ref;
4213         unsigned int i, var_ref_idx;
4214         unsigned int field_pos = 0;
4215         struct synth_event *event;
4216         char *synth_event_name;
4217         int ret = 0;
4218
4219         lockdep_assert_held(&event_mutex);
4220
4221         if (data->use_trace_keyword)
4222                 synth_event_name = data->synth_event_name;
4223         else
4224                 synth_event_name = data->action_name;
4225
4226         event = find_synth_event(synth_event_name);
4227         if (!event) {
4228                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4229                 return -EINVAL;
4230         }
4231
4232         event->ref++;
4233
4234         var_ref_idx = hist_data->n_var_refs;
4235
4236         for (i = 0; i < data->n_params; i++) {
4237                 char *p;
4238
4239                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4240                 if (!param) {
4241                         ret = -ENOMEM;
4242                         goto err;
4243                 }
4244
4245                 system = strsep(&param, ".");
4246                 if (!param) {
4247                         param = (char *)system;
4248                         system = event_name = NULL;
4249                 } else {
4250                         event_name = strsep(&param, ".");
4251                         if (!param) {
4252                                 kfree(p);
4253                                 ret = -EINVAL;
4254                                 goto err;
4255                         }
4256                 }
4257
4258                 if (param[0] == '$')
4259                         hist_field = trace_action_find_var(hist_data, data,
4260                                                            system, event_name,
4261                                                            param);
4262                 else
4263                         hist_field = trace_action_create_field_var(hist_data,
4264                                                                    data,
4265                                                                    system,
4266                                                                    event_name,
4267                                                                    param);
4268
4269                 if (!hist_field) {
4270                         kfree(p);
4271                         ret = -EINVAL;
4272                         goto err;
4273                 }
4274
4275                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4276                         var_ref = create_var_ref(hist_data, hist_field,
4277                                                  system, event_name);
4278                         if (!var_ref) {
4279                                 kfree(p);
4280                                 ret = -ENOMEM;
4281                                 goto err;
4282                         }
4283
4284                         field_pos++;
4285                         kfree(p);
4286                         continue;
4287                 }
4288
4289                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4290                 kfree(p);
4291                 ret = -EINVAL;
4292                 goto err;
4293         }
4294
4295         if (field_pos != event->n_fields) {
4296                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4297                 ret = -EINVAL;
4298                 goto err;
4299         }
4300
4301         data->synth_event = event;
4302         data->var_ref_idx = var_ref_idx;
4303  out:
4304         return ret;
4305  err:
4306         event->ref--;
4307
4308         goto out;
4309 }
4310
4311 static int action_create(struct hist_trigger_data *hist_data,
4312                          struct action_data *data)
4313 {
4314         struct trace_event_file *file = hist_data->event_file;
4315         struct trace_array *tr = file->tr;
4316         struct track_data *track_data;
4317         struct field_var *field_var;
4318         unsigned int i;
4319         char *param;
4320         int ret = 0;
4321
4322         if (data->action == ACTION_TRACE)
4323                 return trace_action_create(hist_data, data);
4324
4325         if (data->action == ACTION_SNAPSHOT) {
4326                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4327                 if (IS_ERR(track_data)) {
4328                         ret = PTR_ERR(track_data);
4329                         goto out;
4330                 }
4331
4332                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4333                                                    cond_snapshot_update);
4334                 if (ret)
4335                         track_data_free(track_data);
4336
4337                 goto out;
4338         }
4339
4340         if (data->action == ACTION_SAVE) {
4341                 if (hist_data->n_save_vars) {
4342                         ret = -EEXIST;
4343                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4344                         goto out;
4345                 }
4346
4347                 for (i = 0; i < data->n_params; i++) {
4348                         param = kstrdup(data->params[i], GFP_KERNEL);
4349                         if (!param) {
4350                                 ret = -ENOMEM;
4351                                 goto out;
4352                         }
4353
4354                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4355                         if (IS_ERR(field_var)) {
4356                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4357                                          errpos(param));
4358                                 ret = PTR_ERR(field_var);
4359                                 kfree(param);
4360                                 goto out;
4361                         }
4362
4363                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4364                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4365                                 hist_data->n_save_var_str++;
4366                         kfree(param);
4367                 }
4368         }
4369  out:
4370         return ret;
4371 }
4372
4373 static int onmatch_create(struct hist_trigger_data *hist_data,
4374                           struct action_data *data)
4375 {
4376         return action_create(hist_data, data);
4377 }
4378
4379 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4380 {
4381         char *match_event, *match_event_system;
4382         struct action_data *data;
4383         int ret = -EINVAL;
4384
4385         data = kzalloc(sizeof(*data), GFP_KERNEL);
4386         if (!data)
4387                 return ERR_PTR(-ENOMEM);
4388
4389         match_event = strsep(&str, ")");
4390         if (!match_event || !str) {
4391                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4392                 goto free;
4393         }
4394
4395         match_event_system = strsep(&match_event, ".");
4396         if (!match_event) {
4397                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4398                 goto free;
4399         }
4400
4401         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4402                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4403                 goto free;
4404         }
4405
4406         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4407         if (!data->match_data.event) {
4408                 ret = -ENOMEM;
4409                 goto free;
4410         }
4411
4412         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4413         if (!data->match_data.event_system) {
4414                 ret = -ENOMEM;
4415                 goto free;
4416         }
4417
4418         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4419         if (ret)
4420                 goto free;
4421  out:
4422         return data;
4423  free:
4424         onmatch_destroy(data);
4425         data = ERR_PTR(ret);
4426         goto out;
4427 }
4428
4429 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4430 {
4431         hist_data->fields[HITCOUNT_IDX] =
4432                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4433         if (!hist_data->fields[HITCOUNT_IDX])
4434                 return -ENOMEM;
4435
4436         hist_data->n_vals++;
4437         hist_data->n_fields++;
4438
4439         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4440                 return -EINVAL;
4441
4442         return 0;
4443 }
4444
4445 static int __create_val_field(struct hist_trigger_data *hist_data,
4446                               unsigned int val_idx,
4447                               struct trace_event_file *file,
4448                               char *var_name, char *field_str,
4449                               unsigned long flags)
4450 {
4451         struct hist_field *hist_field;
4452         int ret = 0;
4453
4454         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4455         if (IS_ERR(hist_field)) {
4456                 ret = PTR_ERR(hist_field);
4457                 goto out;
4458         }
4459
4460         hist_data->fields[val_idx] = hist_field;
4461
4462         ++hist_data->n_vals;
4463         ++hist_data->n_fields;
4464
4465         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4466                 ret = -EINVAL;
4467  out:
4468         return ret;
4469 }
4470
4471 static int create_val_field(struct hist_trigger_data *hist_data,
4472                             unsigned int val_idx,
4473                             struct trace_event_file *file,
4474                             char *field_str)
4475 {
4476         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4477                 return -EINVAL;
4478
4479         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4480 }
4481
4482 static int create_var_field(struct hist_trigger_data *hist_data,
4483                             unsigned int val_idx,
4484                             struct trace_event_file *file,
4485                             char *var_name, char *expr_str)
4486 {
4487         struct trace_array *tr = hist_data->event_file->tr;
4488         unsigned long flags = 0;
4489
4490         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4491                 return -EINVAL;
4492
4493         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4494                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4495                 return -EINVAL;
4496         }
4497
4498         flags |= HIST_FIELD_FL_VAR;
4499         hist_data->n_vars++;
4500         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4501                 return -EINVAL;
4502
4503         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4504 }
4505
4506 static int create_val_fields(struct hist_trigger_data *hist_data,
4507                              struct trace_event_file *file)
4508 {
4509         char *fields_str, *field_str;
4510         unsigned int i, j = 1;
4511         int ret;
4512
4513         ret = create_hitcount_val(hist_data);
4514         if (ret)
4515                 goto out;
4516
4517         fields_str = hist_data->attrs->vals_str;
4518         if (!fields_str)
4519                 goto out;
4520
4521         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4522                      j < TRACING_MAP_VALS_MAX; i++) {
4523                 field_str = strsep(&fields_str, ",");
4524                 if (!field_str)
4525                         break;
4526
4527                 if (strcmp(field_str, "hitcount") == 0)
4528                         continue;
4529
4530                 ret = create_val_field(hist_data, j++, file, field_str);
4531                 if (ret)
4532                         goto out;
4533         }
4534
4535         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4536                 ret = -EINVAL;
4537  out:
4538         return ret;
4539 }
4540
4541 static int create_key_field(struct hist_trigger_data *hist_data,
4542                             unsigned int key_idx,
4543                             unsigned int key_offset,
4544                             struct trace_event_file *file,
4545                             char *field_str)
4546 {
4547         struct trace_array *tr = hist_data->event_file->tr;
4548         struct hist_field *hist_field = NULL;
4549         unsigned long flags = 0;
4550         unsigned int key_size;
4551         int ret = 0;
4552
4553         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4554                 return -EINVAL;
4555
4556         flags |= HIST_FIELD_FL_KEY;
4557
4558         if (strcmp(field_str, "stacktrace") == 0) {
4559                 flags |= HIST_FIELD_FL_STACKTRACE;
4560                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4561                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4562         } else {
4563                 hist_field = parse_expr(hist_data, file, field_str, flags,
4564                                         NULL, 0);
4565                 if (IS_ERR(hist_field)) {
4566                         ret = PTR_ERR(hist_field);
4567                         goto out;
4568                 }
4569
4570                 if (field_has_hist_vars(hist_field, 0)) {
4571                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4572                         destroy_hist_field(hist_field, 0);
4573                         ret = -EINVAL;
4574                         goto out;
4575                 }
4576
4577                 key_size = hist_field->size;
4578         }
4579
4580         hist_data->fields[key_idx] = hist_field;
4581
4582         key_size = ALIGN(key_size, sizeof(u64));
4583         hist_data->fields[key_idx]->size = key_size;
4584         hist_data->fields[key_idx]->offset = key_offset;
4585
4586         hist_data->key_size += key_size;
4587
4588         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4589                 ret = -EINVAL;
4590                 goto out;
4591         }
4592
4593         hist_data->n_keys++;
4594         hist_data->n_fields++;
4595
4596         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4597                 return -EINVAL;
4598
4599         ret = key_size;
4600  out:
4601         return ret;
4602 }
4603
4604 static int create_key_fields(struct hist_trigger_data *hist_data,
4605                              struct trace_event_file *file)
4606 {
4607         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4608         char *fields_str, *field_str;
4609         int ret = -EINVAL;
4610
4611         fields_str = hist_data->attrs->keys_str;
4612         if (!fields_str)
4613                 goto out;
4614
4615         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4616                 field_str = strsep(&fields_str, ",");
4617                 if (!field_str)
4618                         break;
4619                 ret = create_key_field(hist_data, i, key_offset,
4620                                        file, field_str);
4621                 if (ret < 0)
4622                         goto out;
4623                 key_offset += ret;
4624         }
4625         if (fields_str) {
4626                 ret = -EINVAL;
4627                 goto out;
4628         }
4629         ret = 0;
4630  out:
4631         return ret;
4632 }
4633
4634 static int create_var_fields(struct hist_trigger_data *hist_data,
4635                              struct trace_event_file *file)
4636 {
4637         unsigned int i, j = hist_data->n_vals;
4638         int ret = 0;
4639
4640         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4641
4642         for (i = 0; i < n_vars; i++) {
4643                 char *var_name = hist_data->attrs->var_defs.name[i];
4644                 char *expr = hist_data->attrs->var_defs.expr[i];
4645
4646                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4647                 if (ret)
4648                         goto out;
4649         }
4650  out:
4651         return ret;
4652 }
4653
4654 static void free_var_defs(struct hist_trigger_data *hist_data)
4655 {
4656         unsigned int i;
4657
4658         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4659                 kfree(hist_data->attrs->var_defs.name[i]);
4660                 kfree(hist_data->attrs->var_defs.expr[i]);
4661         }
4662
4663         hist_data->attrs->var_defs.n_vars = 0;
4664 }
4665
4666 static int parse_var_defs(struct hist_trigger_data *hist_data)
4667 {
4668         struct trace_array *tr = hist_data->event_file->tr;
4669         char *s, *str, *var_name, *field_str;
4670         unsigned int i, j, n_vars = 0;
4671         int ret = 0;
4672
4673         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4674                 str = hist_data->attrs->assignment_str[i];
4675                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4676                         field_str = strsep(&str, ",");
4677                         if (!field_str)
4678                                 break;
4679
4680                         var_name = strsep(&field_str, "=");
4681                         if (!var_name || !field_str) {
4682                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4683                                          errpos(var_name));
4684                                 ret = -EINVAL;
4685                                 goto free;
4686                         }
4687
4688                         if (n_vars == TRACING_MAP_VARS_MAX) {
4689                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4690                                 ret = -EINVAL;
4691                                 goto free;
4692                         }
4693
4694                         s = kstrdup(var_name, GFP_KERNEL);
4695                         if (!s) {
4696                                 ret = -ENOMEM;
4697                                 goto free;
4698                         }
4699                         hist_data->attrs->var_defs.name[n_vars] = s;
4700
4701                         s = kstrdup(field_str, GFP_KERNEL);
4702                         if (!s) {
4703                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4704                                 ret = -ENOMEM;
4705                                 goto free;
4706                         }
4707                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4708
4709                         hist_data->attrs->var_defs.n_vars = n_vars;
4710                 }
4711         }
4712
4713         return ret;
4714  free:
4715         free_var_defs(hist_data);
4716
4717         return ret;
4718 }
4719
4720 static int create_hist_fields(struct hist_trigger_data *hist_data,
4721                               struct trace_event_file *file)
4722 {
4723         int ret;
4724
4725         ret = parse_var_defs(hist_data);
4726         if (ret)
4727                 goto out;
4728
4729         ret = create_val_fields(hist_data, file);
4730         if (ret)
4731                 goto out;
4732
4733         ret = create_var_fields(hist_data, file);
4734         if (ret)
4735                 goto out;
4736
4737         ret = create_key_fields(hist_data, file);
4738         if (ret)
4739                 goto out;
4740  out:
4741         free_var_defs(hist_data);
4742
4743         return ret;
4744 }
4745
4746 static int is_descending(const char *str)
4747 {
4748         if (!str)
4749                 return 0;
4750
4751         if (strcmp(str, "descending") == 0)
4752                 return 1;
4753
4754         if (strcmp(str, "ascending") == 0)
4755                 return 0;
4756
4757         return -EINVAL;
4758 }
4759
4760 static int create_sort_keys(struct hist_trigger_data *hist_data)
4761 {
4762         char *fields_str = hist_data->attrs->sort_key_str;
4763         struct tracing_map_sort_key *sort_key;
4764         int descending, ret = 0;
4765         unsigned int i, j, k;
4766
4767         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4768
4769         if (!fields_str)
4770                 goto out;
4771
4772         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4773                 struct hist_field *hist_field;
4774                 char *field_str, *field_name;
4775                 const char *test_name;
4776
4777                 sort_key = &hist_data->sort_keys[i];
4778
4779                 field_str = strsep(&fields_str, ",");
4780                 if (!field_str)
4781                         break;
4782
4783                 if (!*field_str) {
4784                         ret = -EINVAL;
4785                         break;
4786                 }
4787
4788                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4789                         ret = -EINVAL;
4790                         break;
4791                 }
4792
4793                 field_name = strsep(&field_str, ".");
4794                 if (!field_name || !*field_name) {
4795                         ret = -EINVAL;
4796                         break;
4797                 }
4798
4799                 if (strcmp(field_name, "hitcount") == 0) {
4800                         descending = is_descending(field_str);
4801                         if (descending < 0) {
4802                                 ret = descending;
4803                                 break;
4804                         }
4805                         sort_key->descending = descending;
4806                         continue;
4807                 }
4808
4809                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4810                         unsigned int idx;
4811
4812                         hist_field = hist_data->fields[j];
4813                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4814                                 continue;
4815
4816                         idx = k++;
4817
4818                         test_name = hist_field_name(hist_field, 0);
4819
4820                         if (strcmp(field_name, test_name) == 0) {
4821                                 sort_key->field_idx = idx;
4822                                 descending = is_descending(field_str);
4823                                 if (descending < 0) {
4824                                         ret = descending;
4825                                         goto out;
4826                                 }
4827                                 sort_key->descending = descending;
4828                                 break;
4829                         }
4830                 }
4831                 if (j == hist_data->n_fields) {
4832                         ret = -EINVAL;
4833                         break;
4834                 }
4835         }
4836
4837         hist_data->n_sort_keys = i;
4838  out:
4839         return ret;
4840 }
4841
4842 static void destroy_actions(struct hist_trigger_data *hist_data)
4843 {
4844         unsigned int i;
4845
4846         for (i = 0; i < hist_data->n_actions; i++) {
4847                 struct action_data *data = hist_data->actions[i];
4848
4849                 if (data->handler == HANDLER_ONMATCH)
4850                         onmatch_destroy(data);
4851                 else if (data->handler == HANDLER_ONMAX ||
4852                          data->handler == HANDLER_ONCHANGE)
4853                         track_data_destroy(hist_data, data);
4854                 else
4855                         kfree(data);
4856         }
4857 }
4858
4859 static int parse_actions(struct hist_trigger_data *hist_data)
4860 {
4861         struct trace_array *tr = hist_data->event_file->tr;
4862         struct action_data *data;
4863         unsigned int i;
4864         int ret = 0;
4865         char *str;
4866         int len;
4867
4868         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4869                 str = hist_data->attrs->action_str[i];
4870
4871                 if ((len = str_has_prefix(str, "onmatch("))) {
4872                         char *action_str = str + len;
4873
4874                         data = onmatch_parse(tr, action_str);
4875                         if (IS_ERR(data)) {
4876                                 ret = PTR_ERR(data);
4877                                 break;
4878                         }
4879                 } else if ((len = str_has_prefix(str, "onmax("))) {
4880                         char *action_str = str + len;
4881
4882                         data = track_data_parse(hist_data, action_str,
4883                                                 HANDLER_ONMAX);
4884                         if (IS_ERR(data)) {
4885                                 ret = PTR_ERR(data);
4886                                 break;
4887                         }
4888                 } else if ((len = str_has_prefix(str, "onchange("))) {
4889                         char *action_str = str + len;
4890
4891                         data = track_data_parse(hist_data, action_str,
4892                                                 HANDLER_ONCHANGE);
4893                         if (IS_ERR(data)) {
4894                                 ret = PTR_ERR(data);
4895                                 break;
4896                         }
4897                 } else {
4898                         ret = -EINVAL;
4899                         break;
4900                 }
4901
4902                 hist_data->actions[hist_data->n_actions++] = data;
4903         }
4904
4905         return ret;
4906 }
4907
4908 static int create_actions(struct hist_trigger_data *hist_data)
4909 {
4910         struct action_data *data;
4911         unsigned int i;
4912         int ret = 0;
4913
4914         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4915                 data = hist_data->actions[i];
4916
4917                 if (data->handler == HANDLER_ONMATCH) {
4918                         ret = onmatch_create(hist_data, data);
4919                         if (ret)
4920                                 break;
4921                 } else if (data->handler == HANDLER_ONMAX ||
4922                            data->handler == HANDLER_ONCHANGE) {
4923                         ret = track_data_create(hist_data, data);
4924                         if (ret)
4925                                 break;
4926                 } else {
4927                         ret = -EINVAL;
4928                         break;
4929                 }
4930         }
4931
4932         return ret;
4933 }
4934
4935 static void print_actions(struct seq_file *m,
4936                           struct hist_trigger_data *hist_data,
4937                           struct tracing_map_elt *elt)
4938 {
4939         unsigned int i;
4940
4941         for (i = 0; i < hist_data->n_actions; i++) {
4942                 struct action_data *data = hist_data->actions[i];
4943
4944                 if (data->action == ACTION_SNAPSHOT)
4945                         continue;
4946
4947                 if (data->handler == HANDLER_ONMAX ||
4948                     data->handler == HANDLER_ONCHANGE)
4949                         track_data_print(m, hist_data, elt, data);
4950         }
4951 }
4952
4953 static void print_action_spec(struct seq_file *m,
4954                               struct hist_trigger_data *hist_data,
4955                               struct action_data *data)
4956 {
4957         unsigned int i;
4958
4959         if (data->action == ACTION_SAVE) {
4960                 for (i = 0; i < hist_data->n_save_vars; i++) {
4961                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4962                         if (i < hist_data->n_save_vars - 1)
4963                                 seq_puts(m, ",");
4964                 }
4965         } else if (data->action == ACTION_TRACE) {
4966                 if (data->use_trace_keyword)
4967                         seq_printf(m, "%s", data->synth_event_name);
4968                 for (i = 0; i < data->n_params; i++) {
4969                         if (i || data->use_trace_keyword)
4970                                 seq_puts(m, ",");
4971                         seq_printf(m, "%s", data->params[i]);
4972                 }
4973         }
4974 }
4975
4976 static void print_track_data_spec(struct seq_file *m,
4977                                   struct hist_trigger_data *hist_data,
4978                                   struct action_data *data)
4979 {
4980         if (data->handler == HANDLER_ONMAX)
4981                 seq_puts(m, ":onmax(");
4982         else if (data->handler == HANDLER_ONCHANGE)
4983                 seq_puts(m, ":onchange(");
4984         seq_printf(m, "%s", data->track_data.var_str);
4985         seq_printf(m, ").%s(", data->action_name);
4986
4987         print_action_spec(m, hist_data, data);
4988
4989         seq_puts(m, ")");
4990 }
4991
4992 static void print_onmatch_spec(struct seq_file *m,
4993                                struct hist_trigger_data *hist_data,
4994                                struct action_data *data)
4995 {
4996         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4997                    data->match_data.event);
4998
4999         seq_printf(m, "%s(", data->action_name);
5000
5001         print_action_spec(m, hist_data, data);
5002
5003         seq_puts(m, ")");
5004 }
5005
5006 static bool actions_match(struct hist_trigger_data *hist_data,
5007                           struct hist_trigger_data *hist_data_test)
5008 {
5009         unsigned int i, j;
5010
5011         if (hist_data->n_actions != hist_data_test->n_actions)
5012                 return false;
5013
5014         for (i = 0; i < hist_data->n_actions; i++) {
5015                 struct action_data *data = hist_data->actions[i];
5016                 struct action_data *data_test = hist_data_test->actions[i];
5017                 char *action_name, *action_name_test;
5018
5019                 if (data->handler != data_test->handler)
5020                         return false;
5021                 if (data->action != data_test->action)
5022                         return false;
5023
5024                 if (data->n_params != data_test->n_params)
5025                         return false;
5026
5027                 for (j = 0; j < data->n_params; j++) {
5028                         if (strcmp(data->params[j], data_test->params[j]) != 0)
5029                                 return false;
5030                 }
5031
5032                 if (data->use_trace_keyword)
5033                         action_name = data->synth_event_name;
5034                 else
5035                         action_name = data->action_name;
5036
5037                 if (data_test->use_trace_keyword)
5038                         action_name_test = data_test->synth_event_name;
5039                 else
5040                         action_name_test = data_test->action_name;
5041
5042                 if (strcmp(action_name, action_name_test) != 0)
5043                         return false;
5044
5045                 if (data->handler == HANDLER_ONMATCH) {
5046                         if (strcmp(data->match_data.event_system,
5047                                    data_test->match_data.event_system) != 0)
5048                                 return false;
5049                         if (strcmp(data->match_data.event,
5050                                    data_test->match_data.event) != 0)
5051                                 return false;
5052                 } else if (data->handler == HANDLER_ONMAX ||
5053                            data->handler == HANDLER_ONCHANGE) {
5054                         if (strcmp(data->track_data.var_str,
5055                                    data_test->track_data.var_str) != 0)
5056                                 return false;
5057                 }
5058         }
5059
5060         return true;
5061 }
5062
5063
5064 static void print_actions_spec(struct seq_file *m,
5065                                struct hist_trigger_data *hist_data)
5066 {
5067         unsigned int i;
5068
5069         for (i = 0; i < hist_data->n_actions; i++) {
5070                 struct action_data *data = hist_data->actions[i];
5071
5072                 if (data->handler == HANDLER_ONMATCH)
5073                         print_onmatch_spec(m, hist_data, data);
5074                 else if (data->handler == HANDLER_ONMAX ||
5075                          data->handler == HANDLER_ONCHANGE)
5076                         print_track_data_spec(m, hist_data, data);
5077         }
5078 }
5079
5080 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5081 {
5082         unsigned int i;
5083
5084         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5085                 kfree(hist_data->field_var_hists[i]->cmd);
5086                 kfree(hist_data->field_var_hists[i]);
5087         }
5088 }
5089
5090 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5091 {
5092         if (!hist_data)
5093                 return;
5094
5095         destroy_hist_trigger_attrs(hist_data->attrs);
5096         destroy_hist_fields(hist_data);
5097         tracing_map_destroy(hist_data->map);
5098
5099         destroy_actions(hist_data);
5100         destroy_field_vars(hist_data);
5101         destroy_field_var_hists(hist_data);
5102
5103         kfree(hist_data);
5104 }
5105
5106 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5107 {
5108         struct tracing_map *map = hist_data->map;
5109         struct ftrace_event_field *field;
5110         struct hist_field *hist_field;
5111         int i, idx = 0;
5112
5113         for_each_hist_field(i, hist_data) {
5114                 hist_field = hist_data->fields[i];
5115                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5116                         tracing_map_cmp_fn_t cmp_fn;
5117
5118                         field = hist_field->field;
5119
5120                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5121                                 cmp_fn = tracing_map_cmp_none;
5122                         else if (!field)
5123                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5124                                                              hist_field->is_signed);
5125                         else if (is_string_field(field))
5126                                 cmp_fn = tracing_map_cmp_string;
5127                         else
5128                                 cmp_fn = tracing_map_cmp_num(field->size,
5129                                                              field->is_signed);
5130                         idx = tracing_map_add_key_field(map,
5131                                                         hist_field->offset,
5132                                                         cmp_fn);
5133                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5134                         idx = tracing_map_add_sum_field(map);
5135
5136                 if (idx < 0)
5137                         return idx;
5138
5139                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5140                         idx = tracing_map_add_var(map);
5141                         if (idx < 0)
5142                                 return idx;
5143                         hist_field->var.idx = idx;
5144                         hist_field->var.hist_data = hist_data;
5145                 }
5146         }
5147
5148         return 0;
5149 }
5150
5151 static struct hist_trigger_data *
5152 create_hist_data(unsigned int map_bits,
5153                  struct hist_trigger_attrs *attrs,
5154                  struct trace_event_file *file,
5155                  bool remove)
5156 {
5157         const struct tracing_map_ops *map_ops = NULL;
5158         struct hist_trigger_data *hist_data;
5159         int ret = 0;
5160
5161         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5162         if (!hist_data)
5163                 return ERR_PTR(-ENOMEM);
5164
5165         hist_data->attrs = attrs;
5166         hist_data->remove = remove;
5167         hist_data->event_file = file;
5168
5169         ret = parse_actions(hist_data);
5170         if (ret)
5171                 goto free;
5172
5173         ret = create_hist_fields(hist_data, file);
5174         if (ret)
5175                 goto free;
5176
5177         ret = create_sort_keys(hist_data);
5178         if (ret)
5179                 goto free;
5180
5181         map_ops = &hist_trigger_elt_data_ops;
5182
5183         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5184                                             map_ops, hist_data);
5185         if (IS_ERR(hist_data->map)) {
5186                 ret = PTR_ERR(hist_data->map);
5187                 hist_data->map = NULL;
5188                 goto free;
5189         }
5190
5191         ret = create_tracing_map_fields(hist_data);
5192         if (ret)
5193                 goto free;
5194  out:
5195         return hist_data;
5196  free:
5197         hist_data->attrs = NULL;
5198
5199         destroy_hist_data(hist_data);
5200
5201         hist_data = ERR_PTR(ret);
5202
5203         goto out;
5204 }
5205
5206 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5207                                     struct tracing_map_elt *elt, void *rec,
5208                                     struct ring_buffer_event *rbe,
5209                                     u64 *var_ref_vals)
5210 {
5211         struct hist_elt_data *elt_data;
5212         struct hist_field *hist_field;
5213         unsigned int i, var_idx;
5214         u64 hist_val;
5215
5216         elt_data = elt->private_data;
5217         elt_data->var_ref_vals = var_ref_vals;
5218
5219         for_each_hist_val_field(i, hist_data) {
5220                 hist_field = hist_data->fields[i];
5221                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5222                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5223                         var_idx = hist_field->var.idx;
5224                         tracing_map_set_var(elt, var_idx, hist_val);
5225                         continue;
5226                 }
5227                 tracing_map_update_sum(elt, i, hist_val);
5228         }
5229
5230         for_each_hist_key_field(i, hist_data) {
5231                 hist_field = hist_data->fields[i];
5232                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5233                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5234                         var_idx = hist_field->var.idx;
5235                         tracing_map_set_var(elt, var_idx, hist_val);
5236                 }
5237         }
5238
5239         update_field_vars(hist_data, elt, rbe, rec);
5240 }
5241
5242 static inline void add_to_key(char *compound_key, void *key,
5243                               struct hist_field *key_field, void *rec)
5244 {
5245         size_t size = key_field->size;
5246
5247         if (key_field->flags & HIST_FIELD_FL_STRING) {
5248                 struct ftrace_event_field *field;
5249
5250                 field = key_field->field;
5251                 if (field->filter_type == FILTER_DYN_STRING)
5252                         size = *(u32 *)(rec + field->offset) >> 16;
5253                 else if (field->filter_type == FILTER_PTR_STRING)
5254                         size = strlen(key);
5255                 else if (field->filter_type == FILTER_STATIC_STRING)
5256                         size = field->size;
5257
5258                 /* ensure NULL-termination */
5259                 if (size > key_field->size - 1)
5260                         size = key_field->size - 1;
5261
5262                 strncpy(compound_key + key_field->offset, (char *)key, size);
5263         } else
5264                 memcpy(compound_key + key_field->offset, key, size);
5265 }
5266
5267 static void
5268 hist_trigger_actions(struct hist_trigger_data *hist_data,
5269                      struct tracing_map_elt *elt, void *rec,
5270                      struct ring_buffer_event *rbe, void *key,
5271                      u64 *var_ref_vals)
5272 {
5273         struct action_data *data;
5274         unsigned int i;
5275
5276         for (i = 0; i < hist_data->n_actions; i++) {
5277                 data = hist_data->actions[i];
5278                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
5279         }
5280 }
5281
5282 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
5283                                struct ring_buffer_event *rbe)
5284 {
5285         struct hist_trigger_data *hist_data = data->private_data;
5286         bool use_compound_key = (hist_data->n_keys > 1);
5287         unsigned long entries[HIST_STACKTRACE_DEPTH];
5288         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5289         char compound_key[HIST_KEY_SIZE_MAX];
5290         struct tracing_map_elt *elt = NULL;
5291         struct hist_field *key_field;
5292         u64 field_contents;
5293         void *key = NULL;
5294         unsigned int i;
5295
5296         memset(compound_key, 0, hist_data->key_size);
5297
5298         for_each_hist_key_field(i, hist_data) {
5299                 key_field = hist_data->fields[i];
5300
5301                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5302                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5303                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5304                                          HIST_STACKTRACE_SKIP);
5305                         key = entries;
5306                 } else {
5307                         field_contents = key_field->fn(key_field, elt, rbe, rec);
5308                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5309                                 key = (void *)(unsigned long)field_contents;
5310                                 use_compound_key = true;
5311                         } else
5312                                 key = (void *)&field_contents;
5313                 }
5314
5315                 if (use_compound_key)
5316                         add_to_key(compound_key, key, key_field, rec);
5317         }
5318
5319         if (use_compound_key)
5320                 key = compound_key;
5321
5322         if (hist_data->n_var_refs &&
5323             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5324                 return;
5325
5326         elt = tracing_map_insert(hist_data->map, key);
5327         if (!elt)
5328                 return;
5329
5330         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
5331
5332         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5333                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
5334 }
5335
5336 static void hist_trigger_stacktrace_print(struct seq_file *m,
5337                                           unsigned long *stacktrace_entries,
5338                                           unsigned int max_entries)
5339 {
5340         char str[KSYM_SYMBOL_LEN];
5341         unsigned int spaces = 8;
5342         unsigned int i;
5343
5344         for (i = 0; i < max_entries; i++) {
5345                 if (!stacktrace_entries[i])
5346                         return;
5347
5348                 seq_printf(m, "%*c", 1 + spaces, ' ');
5349                 sprint_symbol(str, stacktrace_entries[i]);
5350                 seq_printf(m, "%s\n", str);
5351         }
5352 }
5353
5354 static void hist_trigger_print_key(struct seq_file *m,
5355                                    struct hist_trigger_data *hist_data,
5356                                    void *key,
5357                                    struct tracing_map_elt *elt)
5358 {
5359         struct hist_field *key_field;
5360         char str[KSYM_SYMBOL_LEN];
5361         bool multiline = false;
5362         const char *field_name;
5363         unsigned int i;
5364         u64 uval;
5365
5366         seq_puts(m, "{ ");
5367
5368         for_each_hist_key_field(i, hist_data) {
5369                 key_field = hist_data->fields[i];
5370
5371                 if (i > hist_data->n_vals)
5372                         seq_puts(m, ", ");
5373
5374                 field_name = hist_field_name(key_field, 0);
5375
5376                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5377                         uval = *(u64 *)(key + key_field->offset);
5378                         seq_printf(m, "%s: %llx", field_name, uval);
5379                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5380                         uval = *(u64 *)(key + key_field->offset);
5381                         sprint_symbol_no_offset(str, uval);
5382                         seq_printf(m, "%s: [%llx] %-45s", field_name,
5383                                    uval, str);
5384                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5385                         uval = *(u64 *)(key + key_field->offset);
5386                         sprint_symbol(str, uval);
5387                         seq_printf(m, "%s: [%llx] %-55s", field_name,
5388                                    uval, str);
5389                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5390                         struct hist_elt_data *elt_data = elt->private_data;
5391                         char *comm;
5392
5393                         if (WARN_ON_ONCE(!elt_data))
5394                                 return;
5395
5396                         comm = elt_data->comm;
5397
5398                         uval = *(u64 *)(key + key_field->offset);
5399                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5400                                    comm, uval);
5401                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5402                         const char *syscall_name;
5403
5404                         uval = *(u64 *)(key + key_field->offset);
5405                         syscall_name = get_syscall_name(uval);
5406                         if (!syscall_name)
5407                                 syscall_name = "unknown_syscall";
5408
5409                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5410                                    syscall_name, uval);
5411                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5412                         seq_puts(m, "stacktrace:\n");
5413                         hist_trigger_stacktrace_print(m,
5414                                                       key + key_field->offset,
5415                                                       HIST_STACKTRACE_DEPTH);
5416                         multiline = true;
5417                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5418                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5419                                    *(u64 *)(key + key_field->offset));
5420                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5421                         seq_printf(m, "%s: %-50s", field_name,
5422                                    (char *)(key + key_field->offset));
5423                 } else {
5424                         uval = *(u64 *)(key + key_field->offset);
5425                         seq_printf(m, "%s: %10llu", field_name, uval);
5426                 }
5427         }
5428
5429         if (!multiline)
5430                 seq_puts(m, " ");
5431
5432         seq_puts(m, "}");
5433 }
5434
5435 static void hist_trigger_entry_print(struct seq_file *m,
5436                                      struct hist_trigger_data *hist_data,
5437                                      void *key,
5438                                      struct tracing_map_elt *elt)
5439 {
5440         const char *field_name;
5441         unsigned int i;
5442
5443         hist_trigger_print_key(m, hist_data, key, elt);
5444
5445         seq_printf(m, " hitcount: %10llu",
5446                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5447
5448         for (i = 1; i < hist_data->n_vals; i++) {
5449                 field_name = hist_field_name(hist_data->fields[i], 0);
5450
5451                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5452                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5453                         continue;
5454
5455                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5456                         seq_printf(m, "  %s: %10llx", field_name,
5457                                    tracing_map_read_sum(elt, i));
5458                 } else {
5459                         seq_printf(m, "  %s: %10llu", field_name,
5460                                    tracing_map_read_sum(elt, i));
5461                 }
5462         }
5463
5464         print_actions(m, hist_data, elt);
5465
5466         seq_puts(m, "\n");
5467 }
5468
5469 static int print_entries(struct seq_file *m,
5470                          struct hist_trigger_data *hist_data)
5471 {
5472         struct tracing_map_sort_entry **sort_entries = NULL;
5473         struct tracing_map *map = hist_data->map;
5474         int i, n_entries;
5475
5476         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5477                                              hist_data->n_sort_keys,
5478                                              &sort_entries);
5479         if (n_entries < 0)
5480                 return n_entries;
5481
5482         for (i = 0; i < n_entries; i++)
5483                 hist_trigger_entry_print(m, hist_data,
5484                                          sort_entries[i]->key,
5485                                          sort_entries[i]->elt);
5486
5487         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5488
5489         return n_entries;
5490 }
5491
5492 static void hist_trigger_show(struct seq_file *m,
5493                               struct event_trigger_data *data, int n)
5494 {
5495         struct hist_trigger_data *hist_data;
5496         int n_entries;
5497
5498         if (n > 0)
5499                 seq_puts(m, "\n\n");
5500
5501         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5502         data->ops->print(m, data->ops, data);
5503         seq_puts(m, "#\n\n");
5504
5505         hist_data = data->private_data;
5506         n_entries = print_entries(m, hist_data);
5507         if (n_entries < 0)
5508                 n_entries = 0;
5509
5510         track_data_snapshot_print(m, hist_data);
5511
5512         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5513                    (u64)atomic64_read(&hist_data->map->hits),
5514                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5515 }
5516
5517 static int hist_show(struct seq_file *m, void *v)
5518 {
5519         struct event_trigger_data *data;
5520         struct trace_event_file *event_file;
5521         int n = 0, ret = 0;
5522
5523         mutex_lock(&event_mutex);
5524
5525         event_file = event_file_data(m->private);
5526         if (unlikely(!event_file)) {
5527                 ret = -ENODEV;
5528                 goto out_unlock;
5529         }
5530
5531         list_for_each_entry(data, &event_file->triggers, list) {
5532                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5533                         hist_trigger_show(m, data, n++);
5534         }
5535
5536  out_unlock:
5537         mutex_unlock(&event_mutex);
5538
5539         return ret;
5540 }
5541
5542 static int event_hist_open(struct inode *inode, struct file *file)
5543 {
5544         int ret;
5545
5546         ret = security_locked_down(LOCKDOWN_TRACEFS);
5547         if (ret)
5548                 return ret;
5549
5550         return single_open(file, hist_show, file);
5551 }
5552
5553 const struct file_operations event_hist_fops = {
5554         .open = event_hist_open,
5555         .read = seq_read,
5556         .llseek = seq_lseek,
5557         .release = single_release,
5558 };
5559
5560 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5561 {
5562         const char *field_name = hist_field_name(hist_field, 0);
5563
5564         if (hist_field->var.name)
5565                 seq_printf(m, "%s=", hist_field->var.name);
5566
5567         if (hist_field->flags & HIST_FIELD_FL_CPU)
5568                 seq_puts(m, "cpu");
5569         else if (field_name) {
5570                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5571                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5572                         seq_putc(m, '$');
5573                 seq_printf(m, "%s", field_name);
5574         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5575                 seq_puts(m, "common_timestamp");
5576
5577         if (hist_field->flags) {
5578                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5579                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5580                         const char *flags = get_hist_field_flags(hist_field);
5581
5582                         if (flags)
5583                                 seq_printf(m, ".%s", flags);
5584                 }
5585         }
5586 }
5587
5588 static int event_hist_trigger_print(struct seq_file *m,
5589                                     struct event_trigger_ops *ops,
5590                                     struct event_trigger_data *data)
5591 {
5592         struct hist_trigger_data *hist_data = data->private_data;
5593         struct hist_field *field;
5594         bool have_var = false;
5595         unsigned int i;
5596
5597         seq_puts(m, "hist:");
5598
5599         if (data->name)
5600                 seq_printf(m, "%s:", data->name);
5601
5602         seq_puts(m, "keys=");
5603
5604         for_each_hist_key_field(i, hist_data) {
5605                 field = hist_data->fields[i];
5606
5607                 if (i > hist_data->n_vals)
5608                         seq_puts(m, ",");
5609
5610                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5611                         seq_puts(m, "stacktrace");
5612                 else
5613                         hist_field_print(m, field);
5614         }
5615
5616         seq_puts(m, ":vals=");
5617
5618         for_each_hist_val_field(i, hist_data) {
5619                 field = hist_data->fields[i];
5620                 if (field->flags & HIST_FIELD_FL_VAR) {
5621                         have_var = true;
5622                         continue;
5623                 }
5624
5625                 if (i == HITCOUNT_IDX)
5626                         seq_puts(m, "hitcount");
5627                 else {
5628                         seq_puts(m, ",");
5629                         hist_field_print(m, field);
5630                 }
5631         }
5632
5633         if (have_var) {
5634                 unsigned int n = 0;
5635
5636                 seq_puts(m, ":");
5637
5638                 for_each_hist_val_field(i, hist_data) {
5639                         field = hist_data->fields[i];
5640
5641                         if (field->flags & HIST_FIELD_FL_VAR) {
5642                                 if (n++)
5643                                         seq_puts(m, ",");
5644                                 hist_field_print(m, field);
5645                         }
5646                 }
5647         }
5648
5649         seq_puts(m, ":sort=");
5650
5651         for (i = 0; i < hist_data->n_sort_keys; i++) {
5652                 struct tracing_map_sort_key *sort_key;
5653                 unsigned int idx, first_key_idx;
5654
5655                 /* skip VAR vals */
5656                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5657
5658                 sort_key = &hist_data->sort_keys[i];
5659                 idx = sort_key->field_idx;
5660
5661                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5662                         return -EINVAL;
5663
5664                 if (i > 0)
5665                         seq_puts(m, ",");
5666
5667                 if (idx == HITCOUNT_IDX)
5668                         seq_puts(m, "hitcount");
5669                 else {
5670                         if (idx >= first_key_idx)
5671                                 idx += hist_data->n_vars;
5672                         hist_field_print(m, hist_data->fields[idx]);
5673                 }
5674
5675                 if (sort_key->descending)
5676                         seq_puts(m, ".descending");
5677         }
5678         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5679         if (hist_data->enable_timestamps)
5680                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5681
5682         print_actions_spec(m, hist_data);
5683
5684         if (data->filter_str)
5685                 seq_printf(m, " if %s", data->filter_str);
5686
5687         if (data->paused)
5688                 seq_puts(m, " [paused]");
5689         else
5690                 seq_puts(m, " [active]");
5691
5692         seq_putc(m, '\n');
5693
5694         return 0;
5695 }
5696
5697 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5698                                    struct event_trigger_data *data)
5699 {
5700         struct hist_trigger_data *hist_data = data->private_data;
5701
5702         if (!data->ref && hist_data->attrs->name)
5703                 save_named_trigger(hist_data->attrs->name, data);
5704
5705         data->ref++;
5706
5707         return 0;
5708 }
5709
5710 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5711 {
5712         struct trace_event_file *file;
5713         unsigned int i;
5714         char *cmd;
5715         int ret;
5716
5717         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5718                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5719                 cmd = hist_data->field_var_hists[i]->cmd;
5720                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5721                                               "!hist", "hist", cmd);
5722         }
5723 }
5724
5725 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5726                                     struct event_trigger_data *data)
5727 {
5728         struct hist_trigger_data *hist_data = data->private_data;
5729
5730         if (WARN_ON_ONCE(data->ref <= 0))
5731                 return;
5732
5733         data->ref--;
5734         if (!data->ref) {
5735                 if (data->name)
5736                         del_named_trigger(data);
5737
5738                 trigger_data_free(data);
5739
5740                 remove_hist_vars(hist_data);
5741
5742                 unregister_field_var_hists(hist_data);
5743
5744                 destroy_hist_data(hist_data);
5745         }
5746 }
5747
5748 static struct event_trigger_ops event_hist_trigger_ops = {
5749         .func                   = event_hist_trigger,
5750         .print                  = event_hist_trigger_print,
5751         .init                   = event_hist_trigger_init,
5752         .free                   = event_hist_trigger_free,
5753 };
5754
5755 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5756                                          struct event_trigger_data *data)
5757 {
5758         data->ref++;
5759
5760         save_named_trigger(data->named_data->name, data);
5761
5762         event_hist_trigger_init(ops, data->named_data);
5763
5764         return 0;
5765 }
5766
5767 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5768                                           struct event_trigger_data *data)
5769 {
5770         if (WARN_ON_ONCE(data->ref <= 0))
5771                 return;
5772
5773         event_hist_trigger_free(ops, data->named_data);
5774
5775         data->ref--;
5776         if (!data->ref) {
5777                 del_named_trigger(data);
5778                 trigger_data_free(data);
5779         }
5780 }
5781
5782 static struct event_trigger_ops event_hist_trigger_named_ops = {
5783         .func                   = event_hist_trigger,
5784         .print                  = event_hist_trigger_print,
5785         .init                   = event_hist_trigger_named_init,
5786         .free                   = event_hist_trigger_named_free,
5787 };
5788
5789 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5790                                                             char *param)
5791 {
5792         return &event_hist_trigger_ops;
5793 }
5794
5795 static void hist_clear(struct event_trigger_data *data)
5796 {
5797         struct hist_trigger_data *hist_data = data->private_data;
5798
5799         if (data->name)
5800                 pause_named_trigger(data);
5801
5802         tracepoint_synchronize_unregister();
5803
5804         tracing_map_clear(hist_data->map);
5805
5806         if (data->name)
5807                 unpause_named_trigger(data);
5808 }
5809
5810 static bool compatible_field(struct ftrace_event_field *field,
5811                              struct ftrace_event_field *test_field)
5812 {
5813         if (field == test_field)
5814                 return true;
5815         if (field == NULL || test_field == NULL)
5816                 return false;
5817         if (strcmp(field->name, test_field->name) != 0)
5818                 return false;
5819         if (strcmp(field->type, test_field->type) != 0)
5820                 return false;
5821         if (field->size != test_field->size)
5822                 return false;
5823         if (field->is_signed != test_field->is_signed)
5824                 return false;
5825
5826         return true;
5827 }
5828
5829 static bool hist_trigger_match(struct event_trigger_data *data,
5830                                struct event_trigger_data *data_test,
5831                                struct event_trigger_data *named_data,
5832                                bool ignore_filter)
5833 {
5834         struct tracing_map_sort_key *sort_key, *sort_key_test;
5835         struct hist_trigger_data *hist_data, *hist_data_test;
5836         struct hist_field *key_field, *key_field_test;
5837         unsigned int i;
5838
5839         if (named_data && (named_data != data_test) &&
5840             (named_data != data_test->named_data))
5841                 return false;
5842
5843         if (!named_data && is_named_trigger(data_test))
5844                 return false;
5845
5846         hist_data = data->private_data;
5847         hist_data_test = data_test->private_data;
5848
5849         if (hist_data->n_vals != hist_data_test->n_vals ||
5850             hist_data->n_fields != hist_data_test->n_fields ||
5851             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5852                 return false;
5853
5854         if (!ignore_filter) {
5855                 if ((data->filter_str && !data_test->filter_str) ||
5856                    (!data->filter_str && data_test->filter_str))
5857                         return false;
5858         }
5859
5860         for_each_hist_field(i, hist_data) {
5861                 key_field = hist_data->fields[i];
5862                 key_field_test = hist_data_test->fields[i];
5863
5864                 if (key_field->flags != key_field_test->flags)
5865                         return false;
5866                 if (!compatible_field(key_field->field, key_field_test->field))
5867                         return false;
5868                 if (key_field->offset != key_field_test->offset)
5869                         return false;
5870                 if (key_field->size != key_field_test->size)
5871                         return false;
5872                 if (key_field->is_signed != key_field_test->is_signed)
5873                         return false;
5874                 if (!!key_field->var.name != !!key_field_test->var.name)
5875                         return false;
5876                 if (key_field->var.name &&
5877                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5878                         return false;
5879         }
5880
5881         for (i = 0; i < hist_data->n_sort_keys; i++) {
5882                 sort_key = &hist_data->sort_keys[i];
5883                 sort_key_test = &hist_data_test->sort_keys[i];
5884
5885                 if (sort_key->field_idx != sort_key_test->field_idx ||
5886                     sort_key->descending != sort_key_test->descending)
5887                         return false;
5888         }
5889
5890         if (!ignore_filter && data->filter_str &&
5891             (strcmp(data->filter_str, data_test->filter_str) != 0))
5892                 return false;
5893
5894         if (!actions_match(hist_data, hist_data_test))
5895                 return false;
5896
5897         return true;
5898 }
5899
5900 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5901                                  struct event_trigger_data *data,
5902                                  struct trace_event_file *file)
5903 {
5904         struct hist_trigger_data *hist_data = data->private_data;
5905         struct event_trigger_data *test, *named_data = NULL;
5906         struct trace_array *tr = file->tr;
5907         int ret = 0;
5908
5909         if (hist_data->attrs->name) {
5910                 named_data = find_named_trigger(hist_data->attrs->name);
5911                 if (named_data) {
5912                         if (!hist_trigger_match(data, named_data, named_data,
5913                                                 true)) {
5914                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5915                                 ret = -EINVAL;
5916                                 goto out;
5917                         }
5918                 }
5919         }
5920
5921         if (hist_data->attrs->name && !named_data)
5922                 goto new;
5923
5924         lockdep_assert_held(&event_mutex);
5925
5926         list_for_each_entry(test, &file->triggers, list) {
5927                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5928                         if (!hist_trigger_match(data, test, named_data, false))
5929                                 continue;
5930                         if (hist_data->attrs->pause)
5931                                 test->paused = true;
5932                         else if (hist_data->attrs->cont)
5933                                 test->paused = false;
5934                         else if (hist_data->attrs->clear)
5935                                 hist_clear(test);
5936                         else {
5937                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5938                                 ret = -EEXIST;
5939                         }
5940                         goto out;
5941                 }
5942         }
5943  new:
5944         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5945                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5946                 ret = -ENOENT;
5947                 goto out;
5948         }
5949
5950         if (hist_data->attrs->pause)
5951                 data->paused = true;
5952
5953         if (named_data) {
5954                 data->private_data = named_data->private_data;
5955                 set_named_trigger_data(data, named_data);
5956                 data->ops = &event_hist_trigger_named_ops;
5957         }
5958
5959         if (data->ops->init) {
5960                 ret = data->ops->init(data->ops, data);
5961                 if (ret < 0)
5962                         goto out;
5963         }
5964
5965         if (hist_data->enable_timestamps) {
5966                 char *clock = hist_data->attrs->clock;
5967
5968                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5969                 if (ret) {
5970                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5971                         goto out;
5972                 }
5973
5974                 tracing_set_time_stamp_abs(file->tr, true);
5975         }
5976
5977         if (named_data)
5978                 destroy_hist_data(hist_data);
5979
5980         ret++;
5981  out:
5982         return ret;
5983 }
5984
5985 static int hist_trigger_enable(struct event_trigger_data *data,
5986                                struct trace_event_file *file)
5987 {
5988         int ret = 0;
5989
5990         list_add_tail_rcu(&data->list, &file->triggers);
5991
5992         update_cond_flag(file);
5993
5994         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5995                 list_del_rcu(&data->list);
5996                 update_cond_flag(file);
5997                 ret--;
5998         }
5999
6000         return ret;
6001 }
6002
6003 static bool have_hist_trigger_match(struct event_trigger_data *data,
6004                                     struct trace_event_file *file)
6005 {
6006         struct hist_trigger_data *hist_data = data->private_data;
6007         struct event_trigger_data *test, *named_data = NULL;
6008         bool match = false;
6009
6010         lockdep_assert_held(&event_mutex);
6011
6012         if (hist_data->attrs->name)
6013                 named_data = find_named_trigger(hist_data->attrs->name);
6014
6015         list_for_each_entry(test, &file->triggers, list) {
6016                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6017                         if (hist_trigger_match(data, test, named_data, false)) {
6018                                 match = true;
6019                                 break;
6020                         }
6021                 }
6022         }
6023
6024         return match;
6025 }
6026
6027 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6028                                     struct trace_event_file *file)
6029 {
6030         struct hist_trigger_data *hist_data = data->private_data;
6031         struct event_trigger_data *test, *named_data = NULL;
6032
6033         lockdep_assert_held(&event_mutex);
6034
6035         if (hist_data->attrs->name)
6036                 named_data = find_named_trigger(hist_data->attrs->name);
6037
6038         list_for_each_entry(test, &file->triggers, list) {
6039                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6040                         if (!hist_trigger_match(data, test, named_data, false))
6041                                 continue;
6042                         hist_data = test->private_data;
6043                         if (check_var_refs(hist_data))
6044                                 return true;
6045                         break;
6046                 }
6047         }
6048
6049         return false;
6050 }
6051
6052 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
6053                                     struct event_trigger_data *data,
6054                                     struct trace_event_file *file)
6055 {
6056         struct hist_trigger_data *hist_data = data->private_data;
6057         struct event_trigger_data *test, *named_data = NULL;
6058         bool unregistered = false;
6059
6060         lockdep_assert_held(&event_mutex);
6061
6062         if (hist_data->attrs->name)
6063                 named_data = find_named_trigger(hist_data->attrs->name);
6064
6065         list_for_each_entry(test, &file->triggers, list) {
6066                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6067                         if (!hist_trigger_match(data, test, named_data, false))
6068                                 continue;
6069                         unregistered = true;
6070                         list_del_rcu(&test->list);
6071                         trace_event_trigger_enable_disable(file, 0);
6072                         update_cond_flag(file);
6073                         break;
6074                 }
6075         }
6076
6077         if (unregistered && test->ops->free)
6078                 test->ops->free(test->ops, test);
6079
6080         if (hist_data->enable_timestamps) {
6081                 if (!hist_data->remove || unregistered)
6082                         tracing_set_time_stamp_abs(file->tr, false);
6083         }
6084 }
6085
6086 static bool hist_file_check_refs(struct trace_event_file *file)
6087 {
6088         struct hist_trigger_data *hist_data;
6089         struct event_trigger_data *test;
6090
6091         lockdep_assert_held(&event_mutex);
6092
6093         list_for_each_entry(test, &file->triggers, list) {
6094                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6095                         hist_data = test->private_data;
6096                         if (check_var_refs(hist_data))
6097                                 return true;
6098                 }
6099         }
6100
6101         return false;
6102 }
6103
6104 static void hist_unreg_all(struct trace_event_file *file)
6105 {
6106         struct event_trigger_data *test, *n;
6107         struct hist_trigger_data *hist_data;
6108         struct synth_event *se;
6109         const char *se_name;
6110
6111         lockdep_assert_held(&event_mutex);
6112
6113         if (hist_file_check_refs(file))
6114                 return;
6115
6116         list_for_each_entry_safe(test, n, &file->triggers, list) {
6117                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6118                         hist_data = test->private_data;
6119                         list_del_rcu(&test->list);
6120                         trace_event_trigger_enable_disable(file, 0);
6121
6122                         se_name = trace_event_name(file->event_call);
6123                         se = find_synth_event(se_name);
6124                         if (se)
6125                                 se->ref--;
6126
6127                         update_cond_flag(file);
6128                         if (hist_data->enable_timestamps)
6129                                 tracing_set_time_stamp_abs(file->tr, false);
6130                         if (test->ops->free)
6131                                 test->ops->free(test->ops, test);
6132                 }
6133         }
6134 }
6135
6136 static int event_hist_trigger_func(struct event_command *cmd_ops,
6137                                    struct trace_event_file *file,
6138                                    char *glob, char *cmd, char *param)
6139 {
6140         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6141         struct event_trigger_data *trigger_data;
6142         struct hist_trigger_attrs *attrs;
6143         struct event_trigger_ops *trigger_ops;
6144         struct hist_trigger_data *hist_data;
6145         struct synth_event *se;
6146         const char *se_name;
6147         bool remove = false;
6148         char *trigger, *p;
6149         int ret = 0;
6150
6151         lockdep_assert_held(&event_mutex);
6152
6153         if (glob && strlen(glob)) {
6154                 hist_err_clear();
6155                 last_cmd_set(file, param);
6156         }
6157
6158         if (!param)
6159                 return -EINVAL;
6160
6161         if (glob[0] == '!')
6162                 remove = true;
6163
6164         /*
6165          * separate the trigger from the filter (k:v [if filter])
6166          * allowing for whitespace in the trigger
6167          */
6168         p = trigger = param;
6169         do {
6170                 p = strstr(p, "if");
6171                 if (!p)
6172                         break;
6173                 if (p == param)
6174                         return -EINVAL;
6175                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6176                         p++;
6177                         continue;
6178                 }
6179                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6180                         return -EINVAL;
6181                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6182                         p++;
6183                         continue;
6184                 }
6185                 break;
6186         } while (p);
6187
6188         if (!p)
6189                 param = NULL;
6190         else {
6191                 *(p - 1) = '\0';
6192                 param = strstrip(p);
6193                 trigger = strstrip(trigger);
6194         }
6195
6196         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6197         if (IS_ERR(attrs))
6198                 return PTR_ERR(attrs);
6199
6200         if (attrs->map_bits)
6201                 hist_trigger_bits = attrs->map_bits;
6202
6203         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6204         if (IS_ERR(hist_data)) {
6205                 destroy_hist_trigger_attrs(attrs);
6206                 return PTR_ERR(hist_data);
6207         }
6208
6209         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6210
6211         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6212         if (!trigger_data) {
6213                 ret = -ENOMEM;
6214                 goto out_free;
6215         }
6216
6217         trigger_data->count = -1;
6218         trigger_data->ops = trigger_ops;
6219         trigger_data->cmd_ops = cmd_ops;
6220
6221         INIT_LIST_HEAD(&trigger_data->list);
6222         RCU_INIT_POINTER(trigger_data->filter, NULL);
6223
6224         trigger_data->private_data = hist_data;
6225
6226         /* if param is non-empty, it's supposed to be a filter */
6227         if (param && cmd_ops->set_filter) {
6228                 ret = cmd_ops->set_filter(param, trigger_data, file);
6229                 if (ret < 0)
6230                         goto out_free;
6231         }
6232
6233         if (remove) {
6234                 if (!have_hist_trigger_match(trigger_data, file))
6235                         goto out_free;
6236
6237                 if (hist_trigger_check_refs(trigger_data, file)) {
6238                         ret = -EBUSY;
6239                         goto out_free;
6240                 }
6241
6242                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6243                 se_name = trace_event_name(file->event_call);
6244                 se = find_synth_event(se_name);
6245                 if (se)
6246                         se->ref--;
6247                 ret = 0;
6248                 goto out_free;
6249         }
6250
6251         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6252         /*
6253          * The above returns on success the # of triggers registered,
6254          * but if it didn't register any it returns zero.  Consider no
6255          * triggers registered a failure too.
6256          */
6257         if (!ret) {
6258                 if (!(attrs->pause || attrs->cont || attrs->clear))
6259                         ret = -ENOENT;
6260                 goto out_free;
6261         } else if (ret < 0)
6262                 goto out_free;
6263
6264         if (get_named_trigger_data(trigger_data))
6265                 goto enable;
6266
6267         if (has_hist_vars(hist_data))
6268                 save_hist_vars(hist_data);
6269
6270         ret = create_actions(hist_data);
6271         if (ret)
6272                 goto out_unreg;
6273
6274         ret = tracing_map_init(hist_data->map);
6275         if (ret)
6276                 goto out_unreg;
6277 enable:
6278         ret = hist_trigger_enable(trigger_data, file);
6279         if (ret)
6280                 goto out_unreg;
6281
6282         se_name = trace_event_name(file->event_call);
6283         se = find_synth_event(se_name);
6284         if (se)
6285                 se->ref++;
6286         /* Just return zero, not the number of registered triggers */
6287         ret = 0;
6288  out:
6289         if (ret == 0)
6290                 hist_err_clear();
6291
6292         return ret;
6293  out_unreg:
6294         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6295  out_free:
6296         if (cmd_ops->set_filter)
6297                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6298
6299         remove_hist_vars(hist_data);
6300
6301         kfree(trigger_data);
6302
6303         destroy_hist_data(hist_data);
6304         goto out;
6305 }
6306
6307 static struct event_command trigger_hist_cmd = {
6308         .name                   = "hist",
6309         .trigger_type           = ETT_EVENT_HIST,
6310         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6311         .func                   = event_hist_trigger_func,
6312         .reg                    = hist_register_trigger,
6313         .unreg                  = hist_unregister_trigger,
6314         .unreg_all              = hist_unreg_all,
6315         .get_trigger_ops        = event_hist_get_trigger_ops,
6316         .set_filter             = set_trigger_filter,
6317 };
6318
6319 __init int register_trigger_hist_cmd(void)
6320 {
6321         int ret;
6322
6323         ret = register_event_command(&trigger_hist_cmd);
6324         WARN_ON(ret < 0);
6325
6326         return ret;
6327 }
6328
6329 static void
6330 hist_enable_trigger(struct event_trigger_data *data, void *rec,
6331                     struct ring_buffer_event *event)
6332 {
6333         struct enable_trigger_data *enable_data = data->private_data;
6334         struct event_trigger_data *test;
6335
6336         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6337                                 lockdep_is_held(&event_mutex)) {
6338                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6339                         if (enable_data->enable)
6340                                 test->paused = false;
6341                         else
6342                                 test->paused = true;
6343                 }
6344         }
6345 }
6346
6347 static void
6348 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
6349                           struct ring_buffer_event *event)
6350 {
6351         if (!data->count)
6352                 return;
6353
6354         if (data->count != -1)
6355                 (data->count)--;
6356
6357         hist_enable_trigger(data, rec, event);
6358 }
6359
6360 static struct event_trigger_ops hist_enable_trigger_ops = {
6361         .func                   = hist_enable_trigger,
6362         .print                  = event_enable_trigger_print,
6363         .init                   = event_trigger_init,
6364         .free                   = event_enable_trigger_free,
6365 };
6366
6367 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6368         .func                   = hist_enable_count_trigger,
6369         .print                  = event_enable_trigger_print,
6370         .init                   = event_trigger_init,
6371         .free                   = event_enable_trigger_free,
6372 };
6373
6374 static struct event_trigger_ops hist_disable_trigger_ops = {
6375         .func                   = hist_enable_trigger,
6376         .print                  = event_enable_trigger_print,
6377         .init                   = event_trigger_init,
6378         .free                   = event_enable_trigger_free,
6379 };
6380
6381 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6382         .func                   = hist_enable_count_trigger,
6383         .print                  = event_enable_trigger_print,
6384         .init                   = event_trigger_init,
6385         .free                   = event_enable_trigger_free,
6386 };
6387
6388 static struct event_trigger_ops *
6389 hist_enable_get_trigger_ops(char *cmd, char *param)
6390 {
6391         struct event_trigger_ops *ops;
6392         bool enable;
6393
6394         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6395
6396         if (enable)
6397                 ops = param ? &hist_enable_count_trigger_ops :
6398                         &hist_enable_trigger_ops;
6399         else
6400                 ops = param ? &hist_disable_count_trigger_ops :
6401                         &hist_disable_trigger_ops;
6402
6403         return ops;
6404 }
6405
6406 static void hist_enable_unreg_all(struct trace_event_file *file)
6407 {
6408         struct event_trigger_data *test, *n;
6409
6410         list_for_each_entry_safe(test, n, &file->triggers, list) {
6411                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6412                         list_del_rcu(&test->list);
6413                         update_cond_flag(file);
6414                         trace_event_trigger_enable_disable(file, 0);
6415                         if (test->ops->free)
6416                                 test->ops->free(test->ops, test);
6417                 }
6418         }
6419 }
6420
6421 static struct event_command trigger_hist_enable_cmd = {
6422         .name                   = ENABLE_HIST_STR,
6423         .trigger_type           = ETT_HIST_ENABLE,
6424         .func                   = event_enable_trigger_func,
6425         .reg                    = event_enable_register_trigger,
6426         .unreg                  = event_enable_unregister_trigger,
6427         .unreg_all              = hist_enable_unreg_all,
6428         .get_trigger_ops        = hist_enable_get_trigger_ops,
6429         .set_filter             = set_trigger_filter,
6430 };
6431
6432 static struct event_command trigger_hist_disable_cmd = {
6433         .name                   = DISABLE_HIST_STR,
6434         .trigger_type           = ETT_HIST_ENABLE,
6435         .func                   = event_enable_trigger_func,
6436         .reg                    = event_enable_register_trigger,
6437         .unreg                  = event_enable_unregister_trigger,
6438         .unreg_all              = hist_enable_unreg_all,
6439         .get_trigger_ops        = hist_enable_get_trigger_ops,
6440         .set_filter             = set_trigger_filter,
6441 };
6442
6443 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6444 {
6445         unregister_event_command(&trigger_hist_enable_cmd);
6446         unregister_event_command(&trigger_hist_disable_cmd);
6447 }
6448
6449 __init int register_trigger_hist_enable_disable_cmds(void)
6450 {
6451         int ret;
6452
6453         ret = register_event_command(&trigger_hist_enable_cmd);
6454         if (WARN_ON(ret < 0))
6455                 return ret;
6456         ret = register_event_command(&trigger_hist_disable_cmd);
6457         if (WARN_ON(ret < 0))
6458                 unregister_trigger_hist_enable_disable_cmds();
6459
6460         return ret;
6461 }
6462
6463 static __init int trace_events_hist_init(void)
6464 {
6465         struct dentry *entry = NULL;
6466         struct dentry *d_tracer;
6467         int err = 0;
6468
6469         err = dyn_event_register(&synth_event_ops);
6470         if (err) {
6471                 pr_warn("Could not register synth_event_ops\n");
6472                 return err;
6473         }
6474
6475         d_tracer = tracing_init_dentry();
6476         if (IS_ERR(d_tracer)) {
6477                 err = PTR_ERR(d_tracer);
6478                 goto err;
6479         }
6480
6481         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
6482                                     NULL, &synth_events_fops);
6483         if (!entry) {
6484                 err = -ENODEV;
6485                 goto err;
6486         }
6487
6488         return err;
6489  err:
6490         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6491
6492         return err;
6493 }
6494
6495 fs_initcall(trace_events_hist_init);