kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace.h"
  23 #include "trace_dynevent.h"
  24
  25 #define SYNTH_SYSTEM            "synthetic"
  26 #define SYNTH_FIELDS_MAX        32
  27
  28 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  29
  30 #define ERRORS                                                          \
  31         C(NONE,                 "No error"),                            \
  32         C(DUPLICATE_VAR,        "Variable already defined"),            \
  33         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  34         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  35         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  36         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  37         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  38         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  39         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  40         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  41         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  42         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  43         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  44         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  45         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  46         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  47         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  48         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  49         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  50         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  51         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  52         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  53         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  54         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  55         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  56         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  57         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  58         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  59         C(INVALID_PARAM,        "Invalid action param"),                \
  60         C(ACTION_NOT_FOUND,     "No action found"),                     \
  61         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  62         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  63         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  64         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  65         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  66         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  67         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  68         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  69         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  70         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  71         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  72         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  73         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  74         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),
  75
  76 #undef C
  77 #define C(a, b)         HIST_ERR_##a
  78
  79 enum { ERRORS };
  80
  81 #undef C
  82 #define C(a, b)         b
  83
  84 static const char *err_text[] = { ERRORS };
  85
  86 struct hist_field;
  87
  88 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  89                                 struct tracing_map_elt *elt,
  90                                 struct ring_buffer_event *rbe,
  91                                 void *event);
  92
  93 #define HIST_FIELD_OPERANDS_MAX 2
  94 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95 #define HIST_ACTIONS_MAX        8
  96
  97 enum field_op_id {
  98         FIELD_OP_NONE,
  99         FIELD_OP_PLUS,
 100         FIELD_OP_MINUS,
 101         FIELD_OP_UNARY_MINUS,
 102 };
 103
 104 /*
 105  * A hist_var (histogram variable) contains variable information for
 106  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 107  * flag set.  A hist_var has a variable name e.g. ts0, and is
 108  * associated with a given histogram trigger, as specified by
 109  * hist_data.  The hist_var idx is the unique index assigned to the
 110  * variable by the hist trigger's tracing_map.  The idx is what is
 111  * used to set a variable's value and, by a variable reference, to
 112  * retrieve it.
 113  */
 114 struct hist_var {
 115         char                            *name;
 116         struct hist_trigger_data        *hist_data;
 117         unsigned int                    idx;
 118 };
 119
 120 struct hist_field {
 121         struct ftrace_event_field       *field;
 122         unsigned long                   flags;
 123         hist_field_fn_t                 fn;
 124         unsigned int                    size;
 125         unsigned int                    offset;
 126         unsigned int                    is_signed;
 127         const char                      *type;
 128         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 129         struct hist_trigger_data        *hist_data;
 130
 131         /*
 132          * Variable fields contain variable-specific info in var.
 133          */
 134         struct hist_var                 var;
 135         enum field_op_id                operator;
 136         char                            *system;
 137         char                            *event_name;
 138
 139         /*
 140          * The name field is used for EXPR and VAR_REF fields.  VAR
 141          * fields contain the variable name in var.name.
 142          */
 143         char                            *name;
 144
 145         /*
 146          * When a histogram trigger is hit, if it has any references
 147          * to variables, the values of those variables are collected
 148          * into a var_ref_vals array by resolve_var_refs().  The
 149          * current value of each variable is read from the tracing_map
 150          * using the hist field's hist_var.idx and entered into the
 151          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 152          */
 153         unsigned int                    var_ref_idx;
 154         bool                            read_once;
 155 };
 156
 157 static u64 hist_field_none(struct hist_field *field,
 158                            struct tracing_map_elt *elt,
 159                            struct ring_buffer_event *rbe,
 160                            void *event)
 161 {
 162         return 0;
 163 }
 164
 165 static u64 hist_field_counter(struct hist_field *field,
 166                               struct tracing_map_elt *elt,
 167                               struct ring_buffer_event *rbe,
 168                               void *event)
 169 {
 170         return 1;
 171 }
 172
 173 static u64 hist_field_string(struct hist_field *hist_field,
 174                              struct tracing_map_elt *elt,
 175                              struct ring_buffer_event *rbe,
 176                              void *event)
 177 {
 178         char *addr = (char *)(event + hist_field->field->offset);
 179
 180         return (u64)(unsigned long)addr;
 181 }
 182
 183 static u64 hist_field_dynstring(struct hist_field *hist_field,
 184                                 struct tracing_map_elt *elt,
 185                                 struct ring_buffer_event *rbe,
 186                                 void *event)
 187 {
 188         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 189         int str_loc = str_item & 0xffff;
 190         char *addr = (char *)(event + str_loc);
 191
 192         return (u64)(unsigned long)addr;
 193 }
 194
 195 static u64 hist_field_pstring(struct hist_field *hist_field,
 196                               struct tracing_map_elt *elt,
 197                               struct ring_buffer_event *rbe,
 198                               void *event)
 199 {
 200         char **addr = (char **)(event + hist_field->field->offset);
 201
 202         return (u64)(unsigned long)*addr;
 203 }
 204
 205 static u64 hist_field_log2(struct hist_field *hist_field,
 206                            struct tracing_map_elt *elt,
 207                            struct ring_buffer_event *rbe,
 208                            void *event)
 209 {
 210         struct hist_field *operand = hist_field->operands[0];
 211
 212         u64 val = operand->fn(operand, elt, rbe, event);
 213
 214         return (u64) ilog2(roundup_pow_of_two(val));
 215 }
 216
 217 static u64 hist_field_plus(struct hist_field *hist_field,
 218                            struct tracing_map_elt *elt,
 219                            struct ring_buffer_event *rbe,
 220                            void *event)
 221 {
 222         struct hist_field *operand1 = hist_field->operands[0];
 223         struct hist_field *operand2 = hist_field->operands[1];
 224
 225         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 226         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 227
 228         return val1 + val2;
 229 }
 230
 231 static u64 hist_field_minus(struct hist_field *hist_field,
 232                             struct tracing_map_elt *elt,
 233                             struct ring_buffer_event *rbe,
 234                             void *event)
 235 {
 236         struct hist_field *operand1 = hist_field->operands[0];
 237         struct hist_field *operand2 = hist_field->operands[1];
 238
 239         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 240         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 241
 242         return val1 - val2;
 243 }
 244
 245 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 246                                   struct tracing_map_elt *elt,
 247                                   struct ring_buffer_event *rbe,
 248                                   void *event)
 249 {
 250         struct hist_field *operand = hist_field->operands[0];
 251
 252         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 253         u64 val = (u64)-sval;
 254
 255         return val;
 256 }
 257
 258 #define DEFINE_HIST_FIELD_FN(type)                                      \
 259         static u64 hist_field_##type(struct hist_field *hist_field,     \
 260                                      struct tracing_map_elt *elt,       \
 261                                      struct ring_buffer_event *rbe,     \
 262                                      void *event)                       \
 263 {                                                                       \
 264         type *addr = (type *)(event + hist_field->field->offset);       \
 265                                                                         \
 266         return (u64)(unsigned long)*addr;                               \
 267 }
 268
 269 DEFINE_HIST_FIELD_FN(s64);
 270 DEFINE_HIST_FIELD_FN(u64);
 271 DEFINE_HIST_FIELD_FN(s32);
 272 DEFINE_HIST_FIELD_FN(u32);
 273 DEFINE_HIST_FIELD_FN(s16);
 274 DEFINE_HIST_FIELD_FN(u16);
 275 DEFINE_HIST_FIELD_FN(s8);
 276 DEFINE_HIST_FIELD_FN(u8);
 277
 278 #define for_each_hist_field(i, hist_data)       \
 279         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 280
 281 #define for_each_hist_val_field(i, hist_data)   \
 282         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 283
 284 #define for_each_hist_key_field(i, hist_data)   \
 285         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 286
 287 #define HIST_STACKTRACE_DEPTH   16
 288 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 289 #define HIST_STACKTRACE_SKIP    5
 290
 291 #define HITCOUNT_IDX            0
 292 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 293
 294 enum hist_field_flags {
 295         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 296         HIST_FIELD_FL_KEY               = 1 << 1,
 297         HIST_FIELD_FL_STRING            = 1 << 2,
 298         HIST_FIELD_FL_HEX               = 1 << 3,
 299         HIST_FIELD_FL_SYM               = 1 << 4,
 300         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 301         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 302         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 303         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 304         HIST_FIELD_FL_LOG2              = 1 << 9,
 305         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 306         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 307         HIST_FIELD_FL_VAR               = 1 << 12,
 308         HIST_FIELD_FL_EXPR              = 1 << 13,
 309         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 310         HIST_FIELD_FL_CPU               = 1 << 15,
 311         HIST_FIELD_FL_ALIAS             = 1 << 16,
 312 };
 313
 314 struct var_defs {
 315         unsigned int    n_vars;
 316         char            *name[TRACING_MAP_VARS_MAX];
 317         char            *expr[TRACING_MAP_VARS_MAX];
 318 };
 319
 320 struct hist_trigger_attrs {
 321         char            *keys_str;
 322         char            *vals_str;
 323         char            *sort_key_str;
 324         char            *name;
 325         char            *clock;
 326         bool            pause;
 327         bool            cont;
 328         bool            clear;
 329         bool            ts_in_usecs;
 330         unsigned int    map_bits;
 331
 332         char            *assignment_str[TRACING_MAP_VARS_MAX];
 333         unsigned int    n_assignments;
 334
 335         char            *action_str[HIST_ACTIONS_MAX];
 336         unsigned int    n_actions;
 337
 338         struct var_defs var_defs;
 339 };
 340
 341 struct field_var {
 342         struct hist_field       *var;
 343         struct hist_field       *val;
 344 };
 345
 346 struct field_var_hist {
 347         struct hist_trigger_data        *hist_data;
 348         char                            *cmd;
 349 };
 350
 351 struct hist_trigger_data {
 352         struct hist_field               *fields[HIST_FIELDS_MAX];
 353         unsigned int                    n_vals;
 354         unsigned int                    n_keys;
 355         unsigned int                    n_fields;
 356         unsigned int                    n_vars;
 357         unsigned int                    key_size;
 358         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 359         unsigned int                    n_sort_keys;
 360         struct trace_event_file         *event_file;
 361         struct hist_trigger_attrs       *attrs;
 362         struct tracing_map              *map;
 363         bool                            enable_timestamps;
 364         bool                            remove;
 365         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 366         unsigned int                    n_var_refs;
 367
 368         struct action_data              *actions[HIST_ACTIONS_MAX];
 369         unsigned int                    n_actions;
 370
 371         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 372         unsigned int                    n_field_vars;
 373         unsigned int                    n_field_var_str;
 374         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 375         unsigned int                    n_field_var_hists;
 376
 377         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 378         unsigned int                    n_save_vars;
 379         unsigned int                    n_save_var_str;
 380 };
 381
 382 static int synth_event_create(int argc, const char **argv);
 383 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 384 static int synth_event_release(struct dyn_event *ev);
 385 static bool synth_event_is_busy(struct dyn_event *ev);
 386 static bool synth_event_match(const char *system, const char *event,
 387                         int argc, const char **argv, struct dyn_event *ev);
 388
 389 static struct dyn_event_operations synth_event_ops = {
 390         .create = synth_event_create,
 391         .show = synth_event_show,
 392         .is_busy = synth_event_is_busy,
 393         .free = synth_event_release,
 394         .match = synth_event_match,
 395 };
 396
 397 struct synth_field {
 398         char *type;
 399         char *name;
 400         size_t size;
 401         bool is_signed;
 402         bool is_string;
 403 };
 404
 405 struct synth_event {
 406         struct dyn_event                        devent;
 407         int                                     ref;
 408         char                                    *name;
 409         struct synth_field                      **fields;
 410         unsigned int                            n_fields;
 411         unsigned int                            n_u64;
 412         struct trace_event_class                class;
 413         struct trace_event_call                 call;
 414         struct tracepoint                       *tp;
 415 };
 416
 417 static bool is_synth_event(struct dyn_event *ev)
 418 {
 419         return ev->ops == &synth_event_ops;
 420 }
 421
 422 static struct synth_event *to_synth_event(struct dyn_event *ev)
 423 {
 424         return container_of(ev, struct synth_event, devent);
 425 }
 426
 427 static bool synth_event_is_busy(struct dyn_event *ev)
 428 {
 429         struct synth_event *event = to_synth_event(ev);
 430
 431         return event->ref != 0;
 432 }
 433
 434 static bool synth_event_match(const char *system, const char *event,
 435                         int argc, const char **argv, struct dyn_event *ev)
 436 {
 437         struct synth_event *sev = to_synth_event(ev);
 438
 439         return strcmp(sev->name, event) == 0 &&
 440                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 441 }
 442
 443 struct action_data;
 444
 445 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 446                              struct tracing_map_elt *elt, void *rec,
 447                              struct ring_buffer_event *rbe, void *key,
 448                              struct action_data *data, u64 *var_ref_vals);
 449
 450 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 451
 452 enum handler_id {
 453         HANDLER_ONMATCH = 1,
 454         HANDLER_ONMAX,
 455         HANDLER_ONCHANGE,
 456 };
 457
 458 enum action_id {
 459         ACTION_SAVE = 1,
 460         ACTION_TRACE,
 461         ACTION_SNAPSHOT,
 462 };
 463
 464 struct action_data {
 465         enum handler_id         handler;
 466         enum action_id          action;
 467         char                    *action_name;
 468         action_fn_t             fn;
 469
 470         unsigned int            n_params;
 471         char                    *params[SYNTH_FIELDS_MAX];
 472
 473         /*
 474          * When a histogram trigger is hit, the values of any
 475          * references to variables, including variables being passed
 476          * as parameters to synthetic events, are collected into a
 477          * var_ref_vals array.  This var_ref_idx is the index of the
 478          * first param in the array to be passed to the synthetic
 479          * event invocation.
 480          */
 481         unsigned int            var_ref_idx;
 482         struct synth_event      *synth_event;
 483         bool                    use_trace_keyword;
 484         char                    *synth_event_name;
 485
 486         union {
 487                 struct {
 488                         char                    *event;
 489                         char                    *event_system;
 490                 } match_data;
 491
 492                 struct {
 493                         /*
 494                          * var_str contains the $-unstripped variable
 495                          * name referenced by var_ref, and used when
 496                          * printing the action.  Because var_ref
 497                          * creation is deferred to create_actions(),
 498                          * we need a per-action way to save it until
 499                          * then, thus var_str.
 500                          */
 501                         char                    *var_str;
 502
 503                         /*
 504                          * var_ref refers to the variable being
 505                          * tracked e.g onmax($var).
 506                          */
 507                         struct hist_field       *var_ref;
 508
 509                         /*
 510                          * track_var contains the 'invisible' tracking
 511                          * variable created to keep the current
 512                          * e.g. max value.
 513                          */
 514                         struct hist_field       *track_var;
 515
 516                         check_track_val_fn_t    check_val;
 517                         action_fn_t             save_data;
 518                 } track_data;
 519         };
 520 };
 521
 522 struct track_data {
 523         u64                             track_val;
 524         bool                            updated;
 525
 526         unsigned int                    key_len;
 527         void                            *key;
 528         struct tracing_map_elt          elt;
 529
 530         struct action_data              *action_data;
 531         struct hist_trigger_data        *hist_data;
 532 };
 533
 534 struct hist_elt_data {
 535         char *comm;
 536         u64 *var_ref_vals;
 537         char *field_var_str[SYNTH_FIELDS_MAX];
 538 };
 539
 540 struct snapshot_context {
 541         struct tracing_map_elt  *elt;
 542         void                    *key;
 543 };
 544
 545 static void track_data_free(struct track_data *track_data)
 546 {
 547         struct hist_elt_data *elt_data;
 548
 549         if (!track_data)
 550                 return;
 551
 552         kfree(track_data->key);
 553
 554         elt_data = track_data->elt.private_data;
 555         if (elt_data) {
 556                 kfree(elt_data->comm);
 557                 kfree(elt_data);
 558         }
 559
 560         kfree(track_data);
 561 }
 562
 563 static struct track_data *track_data_alloc(unsigned int key_len,
 564                                            struct action_data *action_data,
 565                                            struct hist_trigger_data *hist_data)
 566 {
 567         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 568         struct hist_elt_data *elt_data;
 569
 570         if (!data)
 571                 return ERR_PTR(-ENOMEM);
 572
 573         data->key = kzalloc(key_len, GFP_KERNEL);
 574         if (!data->key) {
 575                 track_data_free(data);
 576                 return ERR_PTR(-ENOMEM);
 577         }
 578
 579         data->key_len = key_len;
 580         data->action_data = action_data;
 581         data->hist_data = hist_data;
 582
 583         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 584         if (!elt_data) {
 585                 track_data_free(data);
 586                 return ERR_PTR(-ENOMEM);
 587         }
 588         data->elt.private_data = elt_data;
 589
 590         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 591         if (!elt_data->comm) {
 592                 track_data_free(data);
 593                 return ERR_PTR(-ENOMEM);
 594         }
 595
 596         return data;
 597 }
 598
 599 static char last_cmd[MAX_FILTER_STR_VAL];
 600 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 601
 602 static int errpos(char *str)
 603 {
 604         return err_pos(last_cmd, str);
 605 }
 606
 607 static void last_cmd_set(struct trace_event_file *file, char *str)
 608 {
 609         const char *system = NULL, *name = NULL;
 610         struct trace_event_call *call;
 611
 612         if (!str)
 613                 return;
 614
 615         strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1);
 616
 617         if (file) {
 618                 call = file->event_call;
 619
 620                 system = call->class->system;
 621                 if (system) {
 622                         name = trace_event_name(call);
 623                         if (!name)
 624                                 system = NULL;
 625                 }
 626         }
 627
 628         if (system)
 629                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 630 }
 631
 632 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 633 {
 634         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 635                         err_type, err_pos);
 636 }
 637
 638 static void hist_err_clear(void)
 639 {
 640         last_cmd[0] = '\0';
 641         last_cmd_loc[0] = '\0';
 642 }
 643
 644 struct synth_trace_event {
 645         struct trace_entry      ent;
 646         u64                     fields[];
 647 };
 648
 649 static int synth_event_define_fields(struct trace_event_call *call)
 650 {
 651         struct synth_trace_event trace;
 652         int offset = offsetof(typeof(trace), fields);
 653         struct synth_event *event = call->data;
 654         unsigned int i, size, n_u64;
 655         char *name, *type;
 656         bool is_signed;
 657         int ret = 0;
 658
 659         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 660                 size = event->fields[i]->size;
 661                 is_signed = event->fields[i]->is_signed;
 662                 type = event->fields[i]->type;
 663                 name = event->fields[i]->name;
 664                 ret = trace_define_field(call, type, name, offset, size,
 665                                          is_signed, FILTER_OTHER);
 666                 if (ret)
 667                         break;
 668
 669                 if (event->fields[i]->is_string) {
 670                         offset += STR_VAR_LEN_MAX;
 671                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 672                 } else {
 673                         offset += sizeof(u64);
 674                         n_u64++;
 675                 }
 676         }
 677
 678         event->n_u64 = n_u64;
 679
 680         return ret;
 681 }
 682
 683 static bool synth_field_signed(char *type)
 684 {
 685         if (str_has_prefix(type, "u"))
 686                 return false;
 687         if (strcmp(type, "gfp_t") == 0)
 688                 return false;
 689
 690         return true;
 691 }
 692
 693 static int synth_field_is_string(char *type)
 694 {
 695         if (strstr(type, "char[") != NULL)
 696                 return true;
 697
 698         return false;
 699 }
 700
 701 static int synth_field_string_size(char *type)
 702 {
 703         char buf[4], *end, *start;
 704         unsigned int len;
 705         int size, err;
 706
 707         start = strstr(type, "char[");
 708         if (start == NULL)
 709                 return -EINVAL;
 710         start += sizeof("char[") - 1;
 711
 712         end = strchr(type, ']');
 713         if (!end || end < start)
 714                 return -EINVAL;
 715
 716         len = end - start;
 717         if (len > 3)
 718                 return -EINVAL;
 719
 720         strncpy(buf, start, len);
 721         buf[len] = '\0';
 722
 723         err = kstrtouint(buf, 0, &size);
 724         if (err)
 725                 return err;
 726
 727         if (size > STR_VAR_LEN_MAX)
 728                 return -EINVAL;
 729
 730         return size;
 731 }
 732
 733 static int synth_field_size(char *type)
 734 {
 735         int size = 0;
 736
 737         if (strcmp(type, "s64") == 0)
 738                 size = sizeof(s64);
 739         else if (strcmp(type, "u64") == 0)
 740                 size = sizeof(u64);
 741         else if (strcmp(type, "s32") == 0)
 742                 size = sizeof(s32);
 743         else if (strcmp(type, "u32") == 0)
 744                 size = sizeof(u32);
 745         else if (strcmp(type, "s16") == 0)
 746                 size = sizeof(s16);
 747         else if (strcmp(type, "u16") == 0)
 748                 size = sizeof(u16);
 749         else if (strcmp(type, "s8") == 0)
 750                 size = sizeof(s8);
 751         else if (strcmp(type, "u8") == 0)
 752                 size = sizeof(u8);
 753         else if (strcmp(type, "char") == 0)
 754                 size = sizeof(char);
 755         else if (strcmp(type, "unsigned char") == 0)
 756                 size = sizeof(unsigned char);
 757         else if (strcmp(type, "int") == 0)
 758                 size = sizeof(int);
 759         else if (strcmp(type, "unsigned int") == 0)
 760                 size = sizeof(unsigned int);
 761         else if (strcmp(type, "long") == 0)
 762                 size = sizeof(long);
 763         else if (strcmp(type, "unsigned long") == 0)
 764                 size = sizeof(unsigned long);
 765         else if (strcmp(type, "pid_t") == 0)
 766                 size = sizeof(pid_t);
 767         else if (strcmp(type, "gfp_t") == 0)
 768                 size = sizeof(gfp_t);
 769         else if (synth_field_is_string(type))
 770                 size = synth_field_string_size(type);
 771
 772         return size;
 773 }
 774
 775 static const char *synth_field_fmt(char *type)
 776 {
 777         const char *fmt = "%llu";
 778
 779         if (strcmp(type, "s64") == 0)
 780                 fmt = "%lld";
 781         else if (strcmp(type, "u64") == 0)
 782                 fmt = "%llu";
 783         else if (strcmp(type, "s32") == 0)
 784                 fmt = "%d";
 785         else if (strcmp(type, "u32") == 0)
 786                 fmt = "%u";
 787         else if (strcmp(type, "s16") == 0)
 788                 fmt = "%d";
 789         else if (strcmp(type, "u16") == 0)
 790                 fmt = "%u";
 791         else if (strcmp(type, "s8") == 0)
 792                 fmt = "%d";
 793         else if (strcmp(type, "u8") == 0)
 794                 fmt = "%u";
 795         else if (strcmp(type, "char") == 0)
 796                 fmt = "%d";
 797         else if (strcmp(type, "unsigned char") == 0)
 798                 fmt = "%u";
 799         else if (strcmp(type, "int") == 0)
 800                 fmt = "%d";
 801         else if (strcmp(type, "unsigned int") == 0)
 802                 fmt = "%u";
 803         else if (strcmp(type, "long") == 0)
 804                 fmt = "%ld";
 805         else if (strcmp(type, "unsigned long") == 0)
 806                 fmt = "%lu";
 807         else if (strcmp(type, "pid_t") == 0)
 808                 fmt = "%d";
 809         else if (strcmp(type, "gfp_t") == 0)
 810                 fmt = "%x";
 811         else if (synth_field_is_string(type))
 812                 fmt = "%s";
 813
 814         return fmt;
 815 }
 816
 817 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 818                                            int flags,
 819                                            struct trace_event *event)
 820 {
 821         struct trace_array *tr = iter->tr;
 822         struct trace_seq *s = &iter->seq;
 823         struct synth_trace_event *entry;
 824         struct synth_event *se;
 825         unsigned int i, n_u64;
 826         char print_fmt[32];
 827         const char *fmt;
 828
 829         entry = (struct synth_trace_event *)iter->ent;
 830         se = container_of(event, struct synth_event, call.event);
 831
 832         trace_seq_printf(s, "%s: ", se->name);
 833
 834         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 835                 if (trace_seq_has_overflowed(s))
 836                         goto end;
 837
 838                 fmt = synth_field_fmt(se->fields[i]->type);
 839
 840                 /* parameter types */
 841                 if (tr && tr->trace_flags & TRACE_ITER_VERBOSE)
 842                         trace_seq_printf(s, "%s ", fmt);
 843
 844                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 845
 846                 /* parameter values */
 847                 if (se->fields[i]->is_string) {
 848                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 849                                          (char *)&entry->fields[n_u64],
 850                                          i == se->n_fields - 1 ? "" : " ");
 851                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 852                 } else {
 853                         struct trace_print_flags __flags[] = {
 854                             __def_gfpflag_names, {-1, NULL} };
 855
 856                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 857                                          entry->fields[n_u64],
 858                                          i == se->n_fields - 1 ? "" : " ");
 859
 860                         if (strcmp(se->fields[i]->type, "gfp_t") == 0) {
 861                                 trace_seq_puts(s, " (");
 862                                 trace_print_flags_seq(s, "|",
 863                                                       entry->fields[n_u64],
 864                                                       __flags);
 865                                 trace_seq_putc(s, ')');
 866                         }
 867                         n_u64++;
 868                 }
 869         }
 870 end:
 871         trace_seq_putc(s, '\n');
 872
 873         return trace_handle_return(s);
 874 }
 875
 876 static struct trace_event_functions synth_event_funcs = {
 877         .trace          = print_synth_event
 878 };
 879
 880 static notrace void trace_event_raw_event_synth(void *__data,
 881                                                 u64 *var_ref_vals,
 882                                                 unsigned int var_ref_idx)
 883 {
 884         struct trace_event_file *trace_file = __data;
 885         struct synth_trace_event *entry;
 886         struct trace_event_buffer fbuffer;
 887         struct trace_buffer *buffer;
 888         struct synth_event *event;
 889         unsigned int i, n_u64;
 890         int fields_size = 0;
 891
 892         event = trace_file->event_call->data;
 893
 894         if (trace_trigger_soft_disabled(trace_file))
 895                 return;
 896
 897         fields_size = event->n_u64 * sizeof(u64);
 898
 899         /*
 900          * Avoid ring buffer recursion detection, as this event
 901          * is being performed within another event.
 902          */
 903         buffer = trace_file->tr->array_buffer.buffer;
 904         ring_buffer_nest_start(buffer);
 905
 906         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 907                                            sizeof(*entry) + fields_size);
 908         if (!entry)
 909                 goto out;
 910
 911         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 912                 if (event->fields[i]->is_string) {
 913                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 914                         char *str_field = (char *)&entry->fields[n_u64];
 915
 916                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 917                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 918                 } else {
 919                         struct synth_field *field = event->fields[i];
 920                         u64 val = var_ref_vals[var_ref_idx + i];
 921
 922                         switch (field->size) {
 923                         case 1:
 924                                 *(u8 *)&entry->fields[n_u64] = (u8)val;
 925                                 break;
 926
 927                         case 2:
 928                                 *(u16 *)&entry->fields[n_u64] = (u16)val;
 929                                 break;
 930
 931                         case 4:
 932                                 *(u32 *)&entry->fields[n_u64] = (u32)val;
 933                                 break;
 934
 935                         default:
 936                                 entry->fields[n_u64] = val;
 937                                 break;
 938                         }
 939                         n_u64++;
 940                 }
 941         }
 942
 943         trace_event_buffer_commit(&fbuffer);
 944 out:
 945         ring_buffer_nest_end(buffer);
 946 }
 947
 948 static void free_synth_event_print_fmt(struct trace_event_call *call)
 949 {
 950         if (call) {
 951                 kfree(call->print_fmt);
 952                 call->print_fmt = NULL;
 953         }
 954 }
 955
 956 static int __set_synth_event_print_fmt(struct synth_event *event,
 957                                        char *buf, int len)
 958 {
 959         const char *fmt;
 960         int pos = 0;
 961         int i;
 962
 963         /* When len=0, we just calculate the needed length */
 964 #define LEN_OR_ZERO (len ? len - pos : 0)
 965
 966         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 967         for (i = 0; i < event->n_fields; i++) {
 968                 fmt = synth_field_fmt(event->fields[i]->type);
 969                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 970                                 event->fields[i]->name, fmt,
 971                                 i == event->n_fields - 1 ? "" : ", ");
 972         }
 973         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 974
 975         for (i = 0; i < event->n_fields; i++) {
 976                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 977                                 ", REC->%s", event->fields[i]->name);
 978         }
 979
 980 #undef LEN_OR_ZERO
 981
 982         /* return the length of print_fmt */
 983         return pos;
 984 }
 985
 986 static int set_synth_event_print_fmt(struct trace_event_call *call)
 987 {
 988         struct synth_event *event = call->data;
 989         char *print_fmt;
 990         int len;
 991
 992         /* First: called with 0 length to calculate the needed length */
 993         len = __set_synth_event_print_fmt(event, NULL, 0);
 994
 995         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 996         if (!print_fmt)
 997                 return -ENOMEM;
 998
 999         /* Second: actually write the @print_fmt */
1000         __set_synth_event_print_fmt(event, print_fmt, len + 1);
1001         call->print_fmt = print_fmt;
1002
1003         return 0;
1004 }
1005
1006 static void free_synth_field(struct synth_field *field)
1007 {
1008         kfree(field->type);
1009         kfree(field->name);
1010         kfree(field);
1011 }
1012
1013 static struct synth_field *parse_synth_field(int argc, const char **argv,
1014                                              int *consumed)
1015 {
1016         struct synth_field *field;
1017         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
1018         int len, ret = 0;
1019
1020         if (field_type[0] == ';')
1021                 field_type++;
1022
1023         if (!strcmp(field_type, "unsigned")) {
1024                 if (argc < 3)
1025                         return ERR_PTR(-EINVAL);
1026                 prefix = "unsigned ";
1027                 field_type = argv[1];
1028                 field_name = argv[2];
1029                 *consumed = 3;
1030         } else {
1031                 field_name = argv[1];
1032                 *consumed = 2;
1033         }
1034
1035         field = kzalloc(sizeof(*field), GFP_KERNEL);
1036         if (!field)
1037                 return ERR_PTR(-ENOMEM);
1038
1039         len = strlen(field_name);
1040         array = strchr(field_name, '[');
1041         if (array)
1042                 len -= strlen(array);
1043         else if (field_name[len - 1] == ';')
1044                 len--;
1045
1046         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1047         if (!field->name) {
1048                 ret = -ENOMEM;
1049                 goto free;
1050         }
1051
1052         if (field_type[0] == ';')
1053                 field_type++;
1054         len = strlen(field_type) + 1;
1055         if (array)
1056                 len += strlen(array);
1057         if (prefix)
1058                 len += strlen(prefix);
1059
1060         field->type = kzalloc(len, GFP_KERNEL);
1061         if (!field->type) {
1062                 ret = -ENOMEM;
1063                 goto free;
1064         }
1065         if (prefix)
1066                 strcat(field->type, prefix);
1067         strcat(field->type, field_type);
1068         if (array) {
1069                 strcat(field->type, array);
1070                 if (field->type[len - 1] == ';')
1071                         field->type[len - 1] = '\0';
1072         }
1073
1074         field->size = synth_field_size(field->type);
1075         if (!field->size) {
1076                 ret = -EINVAL;
1077                 goto free;
1078         }
1079
1080         if (synth_field_is_string(field->type))
1081                 field->is_string = true;
1082
1083         field->is_signed = synth_field_signed(field->type);
1084
1085  out:
1086         return field;
1087  free:
1088         free_synth_field(field);
1089         field = ERR_PTR(ret);
1090         goto out;
1091 }
1092
1093 static void free_synth_tracepoint(struct tracepoint *tp)
1094 {
1095         if (!tp)
1096                 return;
1097
1098         kfree(tp->name);
1099         kfree(tp);
1100 }
1101
1102 static struct tracepoint *alloc_synth_tracepoint(char *name)
1103 {
1104         struct tracepoint *tp;
1105
1106         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1107         if (!tp)
1108                 return ERR_PTR(-ENOMEM);
1109
1110         tp->name = kstrdup(name, GFP_KERNEL);
1111         if (!tp->name) {
1112                 kfree(tp);
1113                 return ERR_PTR(-ENOMEM);
1114         }
1115
1116         return tp;
1117 }
1118
1119 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1120                                     unsigned int var_ref_idx);
1121
1122 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1123                                unsigned int var_ref_idx)
1124 {
1125         struct tracepoint *tp = event->tp;
1126
1127         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1128                 struct tracepoint_func *probe_func_ptr;
1129                 synth_probe_func_t probe_func;
1130                 void *__data;
1131
1132                 if (!(cpu_online(raw_smp_processor_id())))
1133                         return;
1134
1135                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1136                 if (probe_func_ptr) {
1137                         do {
1138                                 probe_func = probe_func_ptr->func;
1139                                 __data = probe_func_ptr->data;
1140                                 probe_func(__data, var_ref_vals, var_ref_idx);
1141                         } while ((++probe_func_ptr)->func);
1142                 }
1143         }
1144 }
1145
1146 static struct synth_event *find_synth_event(const char *name)
1147 {
1148         struct dyn_event *pos;
1149         struct synth_event *event;
1150
1151         for_each_dyn_event(pos) {
1152                 if (!is_synth_event(pos))
1153                         continue;
1154                 event = to_synth_event(pos);
1155                 if (strcmp(event->name, name) == 0)
1156                         return event;
1157         }
1158
1159         return NULL;
1160 }
1161
1162 static int register_synth_event(struct synth_event *event)
1163 {
1164         struct trace_event_call *call = &event->call;
1165         int ret = 0;
1166
1167         event->call.class = &event->class;
1168         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1169         if (!event->class.system) {
1170                 ret = -ENOMEM;
1171                 goto out;
1172         }
1173
1174         event->tp = alloc_synth_tracepoint(event->name);
1175         if (IS_ERR(event->tp)) {
1176                 ret = PTR_ERR(event->tp);
1177                 event->tp = NULL;
1178                 goto out;
1179         }
1180
1181         INIT_LIST_HEAD(&call->class->fields);
1182         call->event.funcs = &synth_event_funcs;
1183         call->class->define_fields = synth_event_define_fields;
1184
1185         ret = register_trace_event(&call->event);
1186         if (!ret) {
1187                 ret = -ENODEV;
1188                 goto out;
1189         }
1190         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1191         call->class->reg = trace_event_reg;
1192         call->class->probe = trace_event_raw_event_synth;
1193         call->data = event;
1194         call->tp = event->tp;
1195
1196         ret = trace_add_event_call(call);
1197         if (ret) {
1198                 pr_warn("Failed to register synthetic event: %s\n",
1199                         trace_event_name(call));
1200                 goto err;
1201         }
1202
1203         ret = set_synth_event_print_fmt(call);
1204         if (ret < 0) {
1205                 trace_remove_event_call(call);
1206                 goto err;
1207         }
1208  out:
1209         return ret;
1210  err:
1211         unregister_trace_event(&call->event);
1212         goto out;
1213 }
1214
1215 static int unregister_synth_event(struct synth_event *event)
1216 {
1217         struct trace_event_call *call = &event->call;
1218         int ret;
1219
1220         ret = trace_remove_event_call(call);
1221
1222         return ret;
1223 }
1224
1225 static void free_synth_event(struct synth_event *event)
1226 {
1227         unsigned int i;
1228
1229         if (!event)
1230                 return;
1231
1232         for (i = 0; i < event->n_fields; i++)
1233                 free_synth_field(event->fields[i]);
1234
1235         kfree(event->fields);
1236         kfree(event->name);
1237         kfree(event->class.system);
1238         free_synth_tracepoint(event->tp);
1239         free_synth_event_print_fmt(&event->call);
1240         kfree(event);
1241 }
1242
1243 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1244                                              struct synth_field **fields)
1245 {
1246         struct synth_event *event;
1247         unsigned int i;
1248
1249         event = kzalloc(sizeof(*event), GFP_KERNEL);
1250         if (!event) {
1251                 event = ERR_PTR(-ENOMEM);
1252                 goto out;
1253         }
1254
1255         event->name = kstrdup(name, GFP_KERNEL);
1256         if (!event->name) {
1257                 kfree(event);
1258                 event = ERR_PTR(-ENOMEM);
1259                 goto out;
1260         }
1261
1262         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1263         if (!event->fields) {
1264                 free_synth_event(event);
1265                 event = ERR_PTR(-ENOMEM);
1266                 goto out;
1267         }
1268
1269         dyn_event_init(&event->devent, &synth_event_ops);
1270
1271         for (i = 0; i < n_fields; i++)
1272                 event->fields[i] = fields[i];
1273
1274         event->n_fields = n_fields;
1275  out:
1276         return event;
1277 }
1278
1279 static void action_trace(struct hist_trigger_data *hist_data,
1280                          struct tracing_map_elt *elt, void *rec,
1281                          struct ring_buffer_event *rbe, void *key,
1282                          struct action_data *data, u64 *var_ref_vals)
1283 {
1284         struct synth_event *event = data->synth_event;
1285
1286         trace_synth(event, var_ref_vals, data->var_ref_idx);
1287 }
1288
1289 struct hist_var_data {
1290         struct list_head list;
1291         struct hist_trigger_data *hist_data;
1292 };
1293
1294 static int __create_synth_event(int argc, const char *name, const char **argv)
1295 {
1296         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1297         struct synth_event *event = NULL;
1298         int i, consumed = 0, n_fields = 0, ret = 0;
1299
1300         /*
1301          * Argument syntax:
1302          *  - Add synthetic event: <event_name> field[;field] ...
1303          *  - Remove synthetic event: !<event_name> field[;field] ...
1304          *      where 'field' = type field_name
1305          */
1306
1307         if (name[0] == '\0' || argc < 1)
1308                 return -EINVAL;
1309
1310         mutex_lock(&event_mutex);
1311
1312         event = find_synth_event(name);
1313         if (event) {
1314                 ret = -EEXIST;
1315                 goto out;
1316         }
1317
1318         for (i = 0; i < argc - 1; i++) {
1319                 if (strcmp(argv[i], ";") == 0)
1320                         continue;
1321                 if (n_fields == SYNTH_FIELDS_MAX) {
1322                         ret = -EINVAL;
1323                         goto err;
1324                 }
1325
1326                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1327                 if (IS_ERR(field)) {
1328                         ret = PTR_ERR(field);
1329                         goto err;
1330                 }
1331                 fields[n_fields++] = field;
1332                 i += consumed - 1;
1333         }
1334
1335         if (i < argc && strcmp(argv[i], ";") != 0) {
1336                 ret = -EINVAL;
1337                 goto err;
1338         }
1339
1340         event = alloc_synth_event(name, n_fields, fields);
1341         if (IS_ERR(event)) {
1342                 ret = PTR_ERR(event);
1343                 event = NULL;
1344                 goto err;
1345         }
1346         ret = register_synth_event(event);
1347         if (!ret)
1348                 dyn_event_add(&event->devent);
1349         else
1350                 free_synth_event(event);
1351  out:
1352         mutex_unlock(&event_mutex);
1353
1354         return ret;
1355  err:
1356         for (i = 0; i < n_fields; i++)
1357                 free_synth_field(fields[i]);
1358
1359         goto out;
1360 }
1361
1362 static int create_or_delete_synth_event(int argc, char **argv)
1363 {
1364         const char *name = argv[0];
1365         struct synth_event *event = NULL;
1366         int ret;
1367
1368         /* trace_run_command() ensures argc != 0 */
1369         if (name[0] == '!') {
1370                 mutex_lock(&event_mutex);
1371                 event = find_synth_event(name + 1);
1372                 if (event) {
1373                         if (event->ref)
1374                                 ret = -EBUSY;
1375                         else {
1376                                 ret = unregister_synth_event(event);
1377                                 if (!ret) {
1378                                         dyn_event_remove(&event->devent);
1379                                         free_synth_event(event);
1380                                 }
1381                         }
1382                 } else
1383                         ret = -ENOENT;
1384                 mutex_unlock(&event_mutex);
1385                 return ret;
1386         }
1387
1388         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1389         return ret == -ECANCELED ? -EINVAL : ret;
1390 }
1391
1392 int synth_event_run_command(const char *command)
1393 {
1394         return trace_run_command(command, create_or_delete_synth_event);
1395 }
1396
1397 static int synth_event_create(int argc, const char **argv)
1398 {
1399         const char *name = argv[0];
1400         int len;
1401
1402         if (name[0] != 's' || name[1] != ':')
1403                 return -ECANCELED;
1404         name += 2;
1405
1406         /* This interface accepts group name prefix */
1407         if (strchr(name, '/')) {
1408                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
1409                 if (len == 0)
1410                         return -EINVAL;
1411                 name += len;
1412         }
1413         return __create_synth_event(argc - 1, name, argv + 1);
1414 }
1415
1416 static int synth_event_release(struct dyn_event *ev)
1417 {
1418         struct synth_event *event = to_synth_event(ev);
1419         int ret;
1420
1421         if (event->ref)
1422                 return -EBUSY;
1423
1424         ret = unregister_synth_event(event);
1425         if (ret)
1426                 return ret;
1427
1428         dyn_event_remove(ev);
1429         free_synth_event(event);
1430         return 0;
1431 }
1432
1433 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1434 {
1435         struct synth_field *field;
1436         unsigned int i;
1437
1438         seq_printf(m, "%s\t", event->name);
1439
1440         for (i = 0; i < event->n_fields; i++) {
1441                 field = event->fields[i];
1442
1443                 /* parameter values */
1444                 seq_printf(m, "%s %s%s", field->type, field->name,
1445                            i == event->n_fields - 1 ? "" : "; ");
1446         }
1447
1448         seq_putc(m, '\n');
1449
1450         return 0;
1451 }
1452
1453 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1454 {
1455         struct synth_event *event = to_synth_event(ev);
1456
1457         seq_printf(m, "s:%s/", event->class.system);
1458
1459         return __synth_event_show(m, event);
1460 }
1461
1462 static int synth_events_seq_show(struct seq_file *m, void *v)
1463 {
1464         struct dyn_event *ev = v;
1465
1466         if (!is_synth_event(ev))
1467                 return 0;
1468
1469         return __synth_event_show(m, to_synth_event(ev));
1470 }
1471
1472 static const struct seq_operations synth_events_seq_op = {
1473         .start  = dyn_event_seq_start,
1474         .next   = dyn_event_seq_next,
1475         .stop   = dyn_event_seq_stop,
1476         .show   = synth_events_seq_show,
1477 };
1478
1479 static int synth_events_open(struct inode *inode, struct file *file)
1480 {
1481         int ret;
1482
1483         ret = security_locked_down(LOCKDOWN_TRACEFS);
1484         if (ret)
1485                 return ret;
1486
1487         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1488                 ret = dyn_events_release_all(&synth_event_ops);
1489                 if (ret < 0)
1490                         return ret;
1491         }
1492
1493         return seq_open(file, &synth_events_seq_op);
1494 }
1495
1496 static ssize_t synth_events_write(struct file *file,
1497                                   const char __user *buffer,
1498                                   size_t count, loff_t *ppos)
1499 {
1500         return trace_parse_run_command(file, buffer, count, ppos,
1501                                        create_or_delete_synth_event);
1502 }
1503
1504 static const struct file_operations synth_events_fops = {
1505         .open           = synth_events_open,
1506         .write          = synth_events_write,
1507         .read           = seq_read,
1508         .llseek         = seq_lseek,
1509         .release        = seq_release,
1510 };
1511
1512 static u64 hist_field_timestamp(struct hist_field *hist_field,
1513                                 struct tracing_map_elt *elt,
1514                                 struct ring_buffer_event *rbe,
1515                                 void *event)
1516 {
1517         struct hist_trigger_data *hist_data = hist_field->hist_data;
1518         struct trace_array *tr = hist_data->event_file->tr;
1519
1520         u64 ts = ring_buffer_event_time_stamp(rbe);
1521
1522         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1523                 ts = ns2usecs(ts);
1524
1525         return ts;
1526 }
1527
1528 static u64 hist_field_cpu(struct hist_field *hist_field,
1529                           struct tracing_map_elt *elt,
1530                           struct ring_buffer_event *rbe,
1531                           void *event)
1532 {
1533         int cpu = smp_processor_id();
1534
1535         return cpu;
1536 }
1537
1538 /**
1539  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1540  * @hist_field: The VAR_REF field to check
1541  * @var_data: The hist trigger that owns the variable
1542  * @var_idx: The trigger variable identifier
1543  *
1544  * Check the given VAR_REF field to see whether or not it references
1545  * the given variable associated with the given trigger.
1546  *
1547  * Return: The VAR_REF field if it does reference the variable, NULL if not
1548  */
1549 static struct hist_field *
1550 check_field_for_var_ref(struct hist_field *hist_field,
1551                         struct hist_trigger_data *var_data,
1552                         unsigned int var_idx)
1553 {
1554         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1555
1556         if (hist_field && hist_field->var.idx == var_idx &&
1557             hist_field->var.hist_data == var_data)
1558                 return hist_field;
1559
1560         return NULL;
1561 }
1562
1563 /**
1564  * find_var_ref - Check if a trigger has a reference to a trigger variable
1565  * @hist_data: The hist trigger that might have a reference to the variable
1566  * @var_data: The hist trigger that owns the variable
1567  * @var_idx: The trigger variable identifier
1568  *
1569  * Check the list of var_refs[] on the first hist trigger to see
1570  * whether any of them are references to the variable on the second
1571  * trigger.
1572  *
1573  * Return: The VAR_REF field referencing the variable if so, NULL if not
1574  */
1575 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1576                                        struct hist_trigger_data *var_data,
1577                                        unsigned int var_idx)
1578 {
1579         struct hist_field *hist_field;
1580         unsigned int i;
1581
1582         for (i = 0; i < hist_data->n_var_refs; i++) {
1583                 hist_field = hist_data->var_refs[i];
1584                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1585                         return hist_field;
1586         }
1587
1588         return NULL;
1589 }
1590
1591 /**
1592  * find_any_var_ref - Check if there is a reference to a given trigger variable
1593  * @hist_data: The hist trigger
1594  * @var_idx: The trigger variable identifier
1595  *
1596  * Check to see whether the given variable is currently referenced by
1597  * any other trigger.
1598  *
1599  * The trigger the variable is defined on is explicitly excluded - the
1600  * assumption being that a self-reference doesn't prevent a trigger
1601  * from being removed.
1602  *
1603  * Return: The VAR_REF field referencing the variable if so, NULL if not
1604  */
1605 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1606                                            unsigned int var_idx)
1607 {
1608         struct trace_array *tr = hist_data->event_file->tr;
1609         struct hist_field *found = NULL;
1610         struct hist_var_data *var_data;
1611
1612         list_for_each_entry(var_data, &tr->hist_vars, list) {
1613                 if (var_data->hist_data == hist_data)
1614                         continue;
1615                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1616                 if (found)
1617                         break;
1618         }
1619
1620         return found;
1621 }
1622
1623 /**
1624  * check_var_refs - Check if there is a reference to any of trigger's variables
1625  * @hist_data: The hist trigger
1626  *
1627  * A trigger can define one or more variables.  If any one of them is
1628  * currently referenced by any other trigger, this function will
1629  * determine that.
1630
1631  * Typically used to determine whether or not a trigger can be removed
1632  * - if there are any references to a trigger's variables, it cannot.
1633  *
1634  * Return: True if there is a reference to any of trigger's variables
1635  */
1636 static bool check_var_refs(struct hist_trigger_data *hist_data)
1637 {
1638         struct hist_field *field;
1639         bool found = false;
1640         int i;
1641
1642         for_each_hist_field(i, hist_data) {
1643                 field = hist_data->fields[i];
1644                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1645                         if (find_any_var_ref(hist_data, field->var.idx)) {
1646                                 found = true;
1647                                 break;
1648                         }
1649                 }
1650         }
1651
1652         return found;
1653 }
1654
1655 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1656 {
1657         struct trace_array *tr = hist_data->event_file->tr;
1658         struct hist_var_data *var_data, *found = NULL;
1659
1660         list_for_each_entry(var_data, &tr->hist_vars, list) {
1661                 if (var_data->hist_data == hist_data) {
1662                         found = var_data;
1663                         break;
1664                 }
1665         }
1666
1667         return found;
1668 }
1669
1670 static bool field_has_hist_vars(struct hist_field *hist_field,
1671                                 unsigned int level)
1672 {
1673         int i;
1674
1675         if (level > 3)
1676                 return false;
1677
1678         if (!hist_field)
1679                 return false;
1680
1681         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1682             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1683                 return true;
1684
1685         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1686                 struct hist_field *operand;
1687
1688                 operand = hist_field->operands[i];
1689                 if (field_has_hist_vars(operand, level + 1))
1690                         return true;
1691         }
1692
1693         return false;
1694 }
1695
1696 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1697 {
1698         struct hist_field *hist_field;
1699         int i;
1700
1701         for_each_hist_field(i, hist_data) {
1702                 hist_field = hist_data->fields[i];
1703                 if (field_has_hist_vars(hist_field, 0))
1704                         return true;
1705         }
1706
1707         return false;
1708 }
1709
1710 static int save_hist_vars(struct hist_trigger_data *hist_data)
1711 {
1712         struct trace_array *tr = hist_data->event_file->tr;
1713         struct hist_var_data *var_data;
1714
1715         var_data = find_hist_vars(hist_data);
1716         if (var_data)
1717                 return 0;
1718
1719         if (tracing_check_open_get_tr(tr))
1720                 return -ENODEV;
1721
1722         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1723         if (!var_data) {
1724                 trace_array_put(tr);
1725                 return -ENOMEM;
1726         }
1727
1728         var_data->hist_data = hist_data;
1729         list_add(&var_data->list, &tr->hist_vars);
1730
1731         return 0;
1732 }
1733
1734 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1735 {
1736         struct trace_array *tr = hist_data->event_file->tr;
1737         struct hist_var_data *var_data;
1738
1739         var_data = find_hist_vars(hist_data);
1740         if (!var_data)
1741                 return;
1742
1743         if (WARN_ON(check_var_refs(hist_data)))
1744                 return;
1745
1746         list_del(&var_data->list);
1747
1748         kfree(var_data);
1749
1750         trace_array_put(tr);
1751 }
1752
1753 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1754                                          const char *var_name)
1755 {
1756         struct hist_field *hist_field, *found = NULL;
1757         int i;
1758
1759         for_each_hist_field(i, hist_data) {
1760                 hist_field = hist_data->fields[i];
1761                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1762                     strcmp(hist_field->var.name, var_name) == 0) {
1763                         found = hist_field;
1764                         break;
1765                 }
1766         }
1767
1768         return found;
1769 }
1770
1771 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1772                                    struct trace_event_file *file,
1773                                    const char *var_name)
1774 {
1775         struct hist_trigger_data *test_data;
1776         struct event_trigger_data *test;
1777         struct hist_field *hist_field;
1778
1779         lockdep_assert_held(&event_mutex);
1780
1781         hist_field = find_var_field(hist_data, var_name);
1782         if (hist_field)
1783                 return hist_field;
1784
1785         list_for_each_entry(test, &file->triggers, list) {
1786                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1787                         test_data = test->private_data;
1788                         hist_field = find_var_field(test_data, var_name);
1789                         if (hist_field)
1790                                 return hist_field;
1791                 }
1792         }
1793
1794         return NULL;
1795 }
1796
1797 static struct trace_event_file *find_var_file(struct trace_array *tr,
1798                                               char *system,
1799                                               char *event_name,
1800                                               char *var_name)
1801 {
1802         struct hist_trigger_data *var_hist_data;
1803         struct hist_var_data *var_data;
1804         struct trace_event_file *file, *found = NULL;
1805
1806         if (system)
1807                 return find_event_file(tr, system, event_name);
1808
1809         list_for_each_entry(var_data, &tr->hist_vars, list) {
1810                 var_hist_data = var_data->hist_data;
1811                 file = var_hist_data->event_file;
1812                 if (file == found)
1813                         continue;
1814
1815                 if (find_var_field(var_hist_data, var_name)) {
1816                         if (found) {
1817                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1818                                 return NULL;
1819                         }
1820
1821                         found = file;
1822                 }
1823         }
1824
1825         return found;
1826 }
1827
1828 static struct hist_field *find_file_var(struct trace_event_file *file,
1829                                         const char *var_name)
1830 {
1831         struct hist_trigger_data *test_data;
1832         struct event_trigger_data *test;
1833         struct hist_field *hist_field;
1834
1835         lockdep_assert_held(&event_mutex);
1836
1837         list_for_each_entry(test, &file->triggers, list) {
1838                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1839                         test_data = test->private_data;
1840                         hist_field = find_var_field(test_data, var_name);
1841                         if (hist_field)
1842                                 return hist_field;
1843                 }
1844         }
1845
1846         return NULL;
1847 }
1848
1849 static struct hist_field *
1850 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1851 {
1852         struct trace_array *tr = hist_data->event_file->tr;
1853         struct hist_field *hist_field, *found = NULL;
1854         struct trace_event_file *file;
1855         unsigned int i;
1856
1857         for (i = 0; i < hist_data->n_actions; i++) {
1858                 struct action_data *data = hist_data->actions[i];
1859
1860                 if (data->handler == HANDLER_ONMATCH) {
1861                         char *system = data->match_data.event_system;
1862                         char *event_name = data->match_data.event;
1863
1864                         file = find_var_file(tr, system, event_name, var_name);
1865                         if (!file)
1866                                 continue;
1867                         hist_field = find_file_var(file, var_name);
1868                         if (hist_field) {
1869                                 if (found) {
1870                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1871                                                  errpos(var_name));
1872                                         return ERR_PTR(-EINVAL);
1873                                 }
1874
1875                                 found = hist_field;
1876                         }
1877                 }
1878         }
1879         return found;
1880 }
1881
1882 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1883                                          char *system,
1884                                          char *event_name,
1885                                          char *var_name)
1886 {
1887         struct trace_array *tr = hist_data->event_file->tr;
1888         struct hist_field *hist_field = NULL;
1889         struct trace_event_file *file;
1890
1891         if (!system || !event_name) {
1892                 hist_field = find_match_var(hist_data, var_name);
1893                 if (IS_ERR(hist_field))
1894                         return NULL;
1895                 if (hist_field)
1896                         return hist_field;
1897         }
1898
1899         file = find_var_file(tr, system, event_name, var_name);
1900         if (!file)
1901                 return NULL;
1902
1903         hist_field = find_file_var(file, var_name);
1904
1905         return hist_field;
1906 }
1907
1908 static u64 hist_field_var_ref(struct hist_field *hist_field,
1909                               struct tracing_map_elt *elt,
1910                               struct ring_buffer_event *rbe,
1911                               void *event)
1912 {
1913         struct hist_elt_data *elt_data;
1914         u64 var_val = 0;
1915
1916         if (WARN_ON_ONCE(!elt))
1917                 return var_val;
1918
1919         elt_data = elt->private_data;
1920         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1921
1922         return var_val;
1923 }
1924
1925 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1926                              u64 *var_ref_vals, bool self)
1927 {
1928         struct hist_trigger_data *var_data;
1929         struct tracing_map_elt *var_elt;
1930         struct hist_field *hist_field;
1931         unsigned int i, var_idx;
1932         bool resolved = true;
1933         u64 var_val = 0;
1934
1935         for (i = 0; i < hist_data->n_var_refs; i++) {
1936                 hist_field = hist_data->var_refs[i];
1937                 var_idx = hist_field->var.idx;
1938                 var_data = hist_field->var.hist_data;
1939
1940                 if (var_data == NULL) {
1941                         resolved = false;
1942                         break;
1943                 }
1944
1945                 if ((self && var_data != hist_data) ||
1946                     (!self && var_data == hist_data))
1947                         continue;
1948
1949                 var_elt = tracing_map_lookup(var_data->map, key);
1950                 if (!var_elt) {
1951                         resolved = false;
1952                         break;
1953                 }
1954
1955                 if (!tracing_map_var_set(var_elt, var_idx)) {
1956                         resolved = false;
1957                         break;
1958                 }
1959
1960                 if (self || !hist_field->read_once)
1961                         var_val = tracing_map_read_var(var_elt, var_idx);
1962                 else
1963                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1964
1965                 var_ref_vals[i] = var_val;
1966         }
1967
1968         return resolved;
1969 }
1970
1971 static const char *hist_field_name(struct hist_field *field,
1972                                    unsigned int level)
1973 {
1974         const char *field_name = "";
1975
1976         if (level > 1)
1977                 return field_name;
1978
1979         if (field->field)
1980                 field_name = field->field->name;
1981         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1982                  field->flags & HIST_FIELD_FL_ALIAS)
1983                 field_name = hist_field_name(field->operands[0], ++level);
1984         else if (field->flags & HIST_FIELD_FL_CPU)
1985                 field_name = "cpu";
1986         else if (field->flags & HIST_FIELD_FL_EXPR ||
1987                  field->flags & HIST_FIELD_FL_VAR_REF) {
1988                 if (field->system) {
1989                         static char full_name[MAX_FILTER_STR_VAL];
1990
1991                         strcat(full_name, field->system);
1992                         strcat(full_name, ".");
1993                         strcat(full_name, field->event_name);
1994                         strcat(full_name, ".");
1995                         strcat(full_name, field->name);
1996                         field_name = full_name;
1997                 } else
1998                         field_name = field->name;
1999         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
2000                 field_name = "common_timestamp";
2001
2002         if (field_name == NULL)
2003                 field_name = "";
2004
2005         return field_name;
2006 }
2007
2008 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
2009 {
2010         hist_field_fn_t fn = NULL;
2011
2012         switch (field_size) {
2013         case 8:
2014                 if (field_is_signed)
2015                         fn = hist_field_s64;
2016                 else
2017                         fn = hist_field_u64;
2018                 break;
2019         case 4:
2020                 if (field_is_signed)
2021                         fn = hist_field_s32;
2022                 else
2023                         fn = hist_field_u32;
2024                 break;
2025         case 2:
2026                 if (field_is_signed)
2027                         fn = hist_field_s16;
2028                 else
2029                         fn = hist_field_u16;
2030                 break;
2031         case 1:
2032                 if (field_is_signed)
2033                         fn = hist_field_s8;
2034                 else
2035                         fn = hist_field_u8;
2036                 break;
2037         }
2038
2039         return fn;
2040 }
2041
2042 static int parse_map_size(char *str)
2043 {
2044         unsigned long size, map_bits;
2045         int ret;
2046
2047         ret = kstrtoul(str, 0, &size);
2048         if (ret)
2049                 goto out;
2050
2051         map_bits = ilog2(roundup_pow_of_two(size));
2052         if (map_bits < TRACING_MAP_BITS_MIN ||
2053             map_bits > TRACING_MAP_BITS_MAX)
2054                 ret = -EINVAL;
2055         else
2056                 ret = map_bits;
2057  out:
2058         return ret;
2059 }
2060
2061 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2062 {
2063         unsigned int i;
2064
2065         if (!attrs)
2066                 return;
2067
2068         for (i = 0; i < attrs->n_assignments; i++)
2069                 kfree(attrs->assignment_str[i]);
2070
2071         for (i = 0; i < attrs->n_actions; i++)
2072                 kfree(attrs->action_str[i]);
2073
2074         kfree(attrs->name);
2075         kfree(attrs->sort_key_str);
2076         kfree(attrs->keys_str);
2077         kfree(attrs->vals_str);
2078         kfree(attrs->clock);
2079         kfree(attrs);
2080 }
2081
2082 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2083 {
2084         int ret = -EINVAL;
2085
2086         if (attrs->n_actions >= HIST_ACTIONS_MAX)
2087                 return ret;
2088
2089         if ((str_has_prefix(str, "onmatch(")) ||
2090             (str_has_prefix(str, "onmax(")) ||
2091             (str_has_prefix(str, "onchange("))) {
2092                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2093                 if (!attrs->action_str[attrs->n_actions]) {
2094                         ret = -ENOMEM;
2095                         return ret;
2096                 }
2097                 attrs->n_actions++;
2098                 ret = 0;
2099         }
2100         return ret;
2101 }
2102
2103 static int parse_assignment(struct trace_array *tr,
2104                             char *str, struct hist_trigger_attrs *attrs)
2105 {
2106         int len, ret = 0;
2107
2108         if ((len = str_has_prefix(str, "key=")) ||
2109             (len = str_has_prefix(str, "keys="))) {
2110                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
2111                 if (!attrs->keys_str) {
2112                         ret = -ENOMEM;
2113                         goto out;
2114                 }
2115         } else if ((len = str_has_prefix(str, "val=")) ||
2116                    (len = str_has_prefix(str, "vals=")) ||
2117                    (len = str_has_prefix(str, "values="))) {
2118                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
2119                 if (!attrs->vals_str) {
2120                         ret = -ENOMEM;
2121                         goto out;
2122                 }
2123         } else if ((len = str_has_prefix(str, "sort="))) {
2124                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
2125                 if (!attrs->sort_key_str) {
2126                         ret = -ENOMEM;
2127                         goto out;
2128                 }
2129         } else if (str_has_prefix(str, "name=")) {
2130                 attrs->name = kstrdup(str, GFP_KERNEL);
2131                 if (!attrs->name) {
2132                         ret = -ENOMEM;
2133                         goto out;
2134                 }
2135         } else if ((len = str_has_prefix(str, "clock="))) {
2136                 str += len;
2137
2138                 str = strstrip(str);
2139                 attrs->clock = kstrdup(str, GFP_KERNEL);
2140                 if (!attrs->clock) {
2141                         ret = -ENOMEM;
2142                         goto out;
2143                 }
2144         } else if ((len = str_has_prefix(str, "size="))) {
2145                 int map_bits = parse_map_size(str + len);
2146
2147                 if (map_bits < 0) {
2148                         ret = map_bits;
2149                         goto out;
2150                 }
2151                 attrs->map_bits = map_bits;
2152         } else {
2153                 char *assignment;
2154
2155                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2156                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
2157                         ret = -EINVAL;
2158                         goto out;
2159                 }
2160
2161                 assignment = kstrdup(str, GFP_KERNEL);
2162                 if (!assignment) {
2163                         ret = -ENOMEM;
2164                         goto out;
2165                 }
2166
2167                 attrs->assignment_str[attrs->n_assignments++] = assignment;
2168         }
2169  out:
2170         return ret;
2171 }
2172
2173 static struct hist_trigger_attrs *
2174 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
2175 {
2176         struct hist_trigger_attrs *attrs;
2177         int ret = 0;
2178
2179         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2180         if (!attrs)
2181                 return ERR_PTR(-ENOMEM);
2182
2183         while (trigger_str) {
2184                 char *str = strsep(&trigger_str, ":");
2185                 char *rhs;
2186
2187                 rhs = strchr(str, '=');
2188                 if (rhs) {
2189                         if (!strlen(++rhs)) {
2190                                 ret = -EINVAL;
2191                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
2192                                 goto free;
2193                         }
2194                         ret = parse_assignment(tr, str, attrs);
2195                         if (ret)
2196                                 goto free;
2197                 } else if (strcmp(str, "pause") == 0)
2198                         attrs->pause = true;
2199                 else if ((strcmp(str, "cont") == 0) ||
2200                          (strcmp(str, "continue") == 0))
2201                         attrs->cont = true;
2202                 else if (strcmp(str, "clear") == 0)
2203                         attrs->clear = true;
2204                 else {
2205                         ret = parse_action(str, attrs);
2206                         if (ret)
2207                                 goto free;
2208                 }
2209         }
2210
2211         if (!attrs->keys_str) {
2212                 ret = -EINVAL;
2213                 goto free;
2214         }
2215
2216         if (!attrs->clock) {
2217                 attrs->clock = kstrdup("global", GFP_KERNEL);
2218                 if (!attrs->clock) {
2219                         ret = -ENOMEM;
2220                         goto free;
2221                 }
2222         }
2223
2224         return attrs;
2225  free:
2226         destroy_hist_trigger_attrs(attrs);
2227
2228         return ERR_PTR(ret);
2229 }
2230
2231 static inline void save_comm(char *comm, struct task_struct *task)
2232 {
2233         if (!task->pid) {
2234                 strcpy(comm, "<idle>");
2235                 return;
2236         }
2237
2238         if (WARN_ON_ONCE(task->pid < 0)) {
2239                 strcpy(comm, "<XXX>");
2240                 return;
2241         }
2242
2243         strncpy(comm, task->comm, TASK_COMM_LEN);
2244 }
2245
2246 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2247 {
2248         unsigned int i;
2249
2250         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2251                 kfree(elt_data->field_var_str[i]);
2252
2253         kfree(elt_data->comm);
2254         kfree(elt_data);
2255 }
2256
2257 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2258 {
2259         struct hist_elt_data *elt_data = elt->private_data;
2260
2261         hist_elt_data_free(elt_data);
2262 }
2263
2264 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2265 {
2266         struct hist_trigger_data *hist_data = elt->map->private_data;
2267         unsigned int size = TASK_COMM_LEN;
2268         struct hist_elt_data *elt_data;
2269         struct hist_field *key_field;
2270         unsigned int i, n_str;
2271
2272         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2273         if (!elt_data)
2274                 return -ENOMEM;
2275
2276         for_each_hist_key_field(i, hist_data) {
2277                 key_field = hist_data->fields[i];
2278
2279                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2280                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2281                         if (!elt_data->comm) {
2282                                 kfree(elt_data);
2283                                 return -ENOMEM;
2284                         }
2285                         break;
2286                 }
2287         }
2288
2289         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2290
2291         size = STR_VAR_LEN_MAX;
2292
2293         for (i = 0; i < n_str; i++) {
2294                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2295                 if (!elt_data->field_var_str[i]) {
2296                         hist_elt_data_free(elt_data);
2297                         return -ENOMEM;
2298                 }
2299         }
2300
2301         elt->private_data = elt_data;
2302
2303         return 0;
2304 }
2305
2306 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2307 {
2308         struct hist_elt_data *elt_data = elt->private_data;
2309
2310         if (elt_data->comm)
2311                 save_comm(elt_data->comm, current);
2312 }
2313
2314 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2315         .elt_alloc      = hist_trigger_elt_data_alloc,
2316         .elt_free       = hist_trigger_elt_data_free,
2317         .elt_init       = hist_trigger_elt_data_init,
2318 };
2319
2320 static const char *get_hist_field_flags(struct hist_field *hist_field)
2321 {
2322         const char *flags_str = NULL;
2323
2324         if (hist_field->flags & HIST_FIELD_FL_HEX)
2325                 flags_str = "hex";
2326         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2327                 flags_str = "sym";
2328         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2329                 flags_str = "sym-offset";
2330         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2331                 flags_str = "execname";
2332         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2333                 flags_str = "syscall";
2334         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2335                 flags_str = "log2";
2336         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2337                 flags_str = "usecs";
2338
2339         return flags_str;
2340 }
2341
2342 static void expr_field_str(struct hist_field *field, char *expr)
2343 {
2344         if (field->flags & HIST_FIELD_FL_VAR_REF)
2345                 strcat(expr, "$");
2346
2347         strcat(expr, hist_field_name(field, 0));
2348
2349         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2350                 const char *flags_str = get_hist_field_flags(field);
2351
2352                 if (flags_str) {
2353                         strcat(expr, ".");
2354                         strcat(expr, flags_str);
2355                 }
2356         }
2357 }
2358
2359 static char *expr_str(struct hist_field *field, unsigned int level)
2360 {
2361         char *expr;
2362
2363         if (level > 1)
2364                 return NULL;
2365
2366         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2367         if (!expr)
2368                 return NULL;
2369
2370         if (!field->operands[0]) {
2371                 expr_field_str(field, expr);
2372                 return expr;
2373         }
2374
2375         if (field->operator == FIELD_OP_UNARY_MINUS) {
2376                 char *subexpr;
2377
2378                 strcat(expr, "-(");
2379                 subexpr = expr_str(field->operands[0], ++level);
2380                 if (!subexpr) {
2381                         kfree(expr);
2382                         return NULL;
2383                 }
2384                 strcat(expr, subexpr);
2385                 strcat(expr, ")");
2386
2387                 kfree(subexpr);
2388
2389                 return expr;
2390         }
2391
2392         expr_field_str(field->operands[0], expr);
2393
2394         switch (field->operator) {
2395         case FIELD_OP_MINUS:
2396                 strcat(expr, "-");
2397                 break;
2398         case FIELD_OP_PLUS:
2399                 strcat(expr, "+");
2400                 break;
2401         default:
2402                 kfree(expr);
2403                 return NULL;
2404         }
2405
2406         expr_field_str(field->operands[1], expr);
2407
2408         return expr;
2409 }
2410
2411 static int contains_operator(char *str)
2412 {
2413         enum field_op_id field_op = FIELD_OP_NONE;
2414         char *op;
2415
2416         op = strpbrk(str, "+-");
2417         if (!op)
2418                 return FIELD_OP_NONE;
2419
2420         switch (*op) {
2421         case '-':
2422                 if (*str == '-')
2423                         field_op = FIELD_OP_UNARY_MINUS;
2424                 else
2425                         field_op = FIELD_OP_MINUS;
2426                 break;
2427         case '+':
2428                 field_op = FIELD_OP_PLUS;
2429                 break;
2430         default:
2431                 break;
2432         }
2433
2434         return field_op;
2435 }
2436
2437 static void __destroy_hist_field(struct hist_field *hist_field)
2438 {
2439         kfree(hist_field->var.name);
2440         kfree(hist_field->name);
2441         kfree(hist_field->type);
2442
2443         kfree(hist_field);
2444 }
2445
2446 static void destroy_hist_field(struct hist_field *hist_field,
2447                                unsigned int level)
2448 {
2449         unsigned int i;
2450
2451         if (level > 3)
2452                 return;
2453
2454         if (!hist_field)
2455                 return;
2456
2457         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2458                 return; /* var refs will be destroyed separately */
2459
2460         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2461                 destroy_hist_field(hist_field->operands[i], level + 1);
2462
2463         __destroy_hist_field(hist_field);
2464 }
2465
2466 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2467                                             struct ftrace_event_field *field,
2468                                             unsigned long flags,
2469                                             char *var_name)
2470 {
2471         struct hist_field *hist_field;
2472
2473         if (field && is_function_field(field))
2474                 return NULL;
2475
2476         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2477         if (!hist_field)
2478                 return NULL;
2479
2480         hist_field->hist_data = hist_data;
2481
2482         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2483                 goto out; /* caller will populate */
2484
2485         if (flags & HIST_FIELD_FL_VAR_REF) {
2486                 hist_field->fn = hist_field_var_ref;
2487                 goto out;
2488         }
2489
2490         if (flags & HIST_FIELD_FL_HITCOUNT) {
2491                 hist_field->fn = hist_field_counter;
2492                 hist_field->size = sizeof(u64);
2493                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2494                 if (!hist_field->type)
2495                         goto free;
2496                 goto out;
2497         }
2498
2499         if (flags & HIST_FIELD_FL_STACKTRACE) {
2500                 hist_field->fn = hist_field_none;
2501                 goto out;
2502         }
2503
2504         if (flags & HIST_FIELD_FL_LOG2) {
2505                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2506                 hist_field->fn = hist_field_log2;
2507                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2508                 hist_field->size = hist_field->operands[0]->size;
2509                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2510                 if (!hist_field->type)
2511                         goto free;
2512                 goto out;
2513         }
2514
2515         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2516                 hist_field->fn = hist_field_timestamp;
2517                 hist_field->size = sizeof(u64);
2518                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2519                 if (!hist_field->type)
2520                         goto free;
2521                 goto out;
2522         }
2523
2524         if (flags & HIST_FIELD_FL_CPU) {
2525                 hist_field->fn = hist_field_cpu;
2526                 hist_field->size = sizeof(int);
2527                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2528                 if (!hist_field->type)
2529                         goto free;
2530                 goto out;
2531         }
2532
2533         if (WARN_ON_ONCE(!field))
2534                 goto out;
2535
2536         if (is_string_field(field)) {
2537                 flags |= HIST_FIELD_FL_STRING;
2538
2539                 hist_field->size = MAX_FILTER_STR_VAL;
2540                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2541                 if (!hist_field->type)
2542                         goto free;
2543
2544                 if (field->filter_type == FILTER_STATIC_STRING)
2545                         hist_field->fn = hist_field_string;
2546                 else if (field->filter_type == FILTER_DYN_STRING)
2547                         hist_field->fn = hist_field_dynstring;
2548                 else
2549                         hist_field->fn = hist_field_pstring;
2550         } else {
2551                 hist_field->size = field->size;
2552                 hist_field->is_signed = field->is_signed;
2553                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2554                 if (!hist_field->type)
2555                         goto free;
2556
2557                 hist_field->fn = select_value_fn(field->size,
2558                                                  field->is_signed);
2559                 if (!hist_field->fn) {
2560                         destroy_hist_field(hist_field, 0);
2561                         return NULL;
2562                 }
2563         }
2564  out:
2565         hist_field->field = field;
2566         hist_field->flags = flags;
2567
2568         if (var_name) {
2569                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2570                 if (!hist_field->var.name)
2571                         goto free;
2572         }
2573
2574         return hist_field;
2575  free:
2576         destroy_hist_field(hist_field, 0);
2577         return NULL;
2578 }
2579
2580 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2581 {
2582         unsigned int i;
2583
2584         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2585                 if (hist_data->fields[i]) {
2586                         destroy_hist_field(hist_data->fields[i], 0);
2587                         hist_data->fields[i] = NULL;
2588                 }
2589         }
2590
2591         for (i = 0; i < hist_data->n_var_refs; i++) {
2592                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2593                 __destroy_hist_field(hist_data->var_refs[i]);
2594                 hist_data->var_refs[i] = NULL;
2595         }
2596 }
2597
2598 static int init_var_ref(struct hist_field *ref_field,
2599                         struct hist_field *var_field,
2600                         char *system, char *event_name)
2601 {
2602         int err = 0;
2603
2604         ref_field->var.idx = var_field->var.idx;
2605         ref_field->var.hist_data = var_field->hist_data;
2606         ref_field->size = var_field->size;
2607         ref_field->is_signed = var_field->is_signed;
2608         ref_field->flags |= var_field->flags &
2609                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2610
2611         if (system) {
2612                 ref_field->system = kstrdup(system, GFP_KERNEL);
2613                 if (!ref_field->system)
2614                         return -ENOMEM;
2615         }
2616
2617         if (event_name) {
2618                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2619                 if (!ref_field->event_name) {
2620                         err = -ENOMEM;
2621                         goto free;
2622                 }
2623         }
2624
2625         if (var_field->var.name) {
2626                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2627                 if (!ref_field->name) {
2628                         err = -ENOMEM;
2629                         goto free;
2630                 }
2631         } else if (var_field->name) {
2632                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2633                 if (!ref_field->name) {
2634                         err = -ENOMEM;
2635                         goto free;
2636                 }
2637         }
2638
2639         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2640         if (!ref_field->type) {
2641                 err = -ENOMEM;
2642                 goto free;
2643         }
2644  out:
2645         return err;
2646  free:
2647         kfree(ref_field->system);
2648         kfree(ref_field->event_name);
2649         kfree(ref_field->name);
2650
2651         goto out;
2652 }
2653
2654 /**
2655  * create_var_ref - Create a variable reference and attach it to trigger
2656  * @hist_data: The trigger that will be referencing the variable
2657  * @var_field: The VAR field to create a reference to
2658  * @system: The optional system string
2659  * @event_name: The optional event_name string
2660  *
2661  * Given a variable hist_field, create a VAR_REF hist_field that
2662  * represents a reference to it.
2663  *
2664  * This function also adds the reference to the trigger that
2665  * now references the variable.
2666  *
2667  * Return: The VAR_REF field if successful, NULL if not
2668  */
2669 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2670                                          struct hist_field *var_field,
2671                                          char *system, char *event_name)
2672 {
2673         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2674         struct hist_field *ref_field;
2675
2676         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2677         if (ref_field) {
2678                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2679                         destroy_hist_field(ref_field, 0);
2680                         return NULL;
2681                 }
2682
2683                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2684                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2685         }
2686
2687         return ref_field;
2688 }
2689
2690 static bool is_var_ref(char *var_name)
2691 {
2692         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2693                 return false;
2694
2695         return true;
2696 }
2697
2698 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2699                                  char *var_name)
2700 {
2701         char *name, *field;
2702         unsigned int i;
2703
2704         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2705                 name = hist_data->attrs->var_defs.name[i];
2706
2707                 if (strcmp(var_name, name) == 0) {
2708                         field = hist_data->attrs->var_defs.expr[i];
2709                         if (contains_operator(field) || is_var_ref(field))
2710                                 continue;
2711                         return field;
2712                 }
2713         }
2714
2715         return NULL;
2716 }
2717
2718 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2719                                  char *system, char *event_name,
2720                                  char *var_name)
2721 {
2722         struct trace_event_call *call;
2723
2724         if (system && event_name) {
2725                 call = hist_data->event_file->event_call;
2726
2727                 if (strcmp(system, call->class->system) != 0)
2728                         return NULL;
2729
2730                 if (strcmp(event_name, trace_event_name(call)) != 0)
2731                         return NULL;
2732         }
2733
2734         if (!!system != !!event_name)
2735                 return NULL;
2736
2737         if (!is_var_ref(var_name))
2738                 return NULL;
2739
2740         var_name++;
2741
2742         return field_name_from_var(hist_data, var_name);
2743 }
2744
2745 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2746                                         char *system, char *event_name,
2747                                         char *var_name)
2748 {
2749         struct hist_field *var_field = NULL, *ref_field = NULL;
2750         struct trace_array *tr = hist_data->event_file->tr;
2751
2752         if (!is_var_ref(var_name))
2753                 return NULL;
2754
2755         var_name++;
2756
2757         var_field = find_event_var(hist_data, system, event_name, var_name);
2758         if (var_field)
2759                 ref_field = create_var_ref(hist_data, var_field,
2760                                            system, event_name);
2761
2762         if (!ref_field)
2763                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2764
2765         return ref_field;
2766 }
2767
2768 static struct ftrace_event_field *
2769 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2770             char *field_str, unsigned long *flags)
2771 {
2772         struct ftrace_event_field *field = NULL;
2773         char *field_name, *modifier, *str;
2774         struct trace_array *tr = file->tr;
2775
2776         modifier = str = kstrdup(field_str, GFP_KERNEL);
2777         if (!modifier)
2778                 return ERR_PTR(-ENOMEM);
2779
2780         field_name = strsep(&modifier, ".");
2781         if (modifier) {
2782                 if (strcmp(modifier, "hex") == 0)
2783                         *flags |= HIST_FIELD_FL_HEX;
2784                 else if (strcmp(modifier, "sym") == 0)
2785                         *flags |= HIST_FIELD_FL_SYM;
2786                 else if (strcmp(modifier, "sym-offset") == 0)
2787                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2788                 else if ((strcmp(modifier, "execname") == 0) &&
2789                          (strcmp(field_name, "common_pid") == 0))
2790                         *flags |= HIST_FIELD_FL_EXECNAME;
2791                 else if (strcmp(modifier, "syscall") == 0)
2792                         *flags |= HIST_FIELD_FL_SYSCALL;
2793                 else if (strcmp(modifier, "log2") == 0)
2794                         *flags |= HIST_FIELD_FL_LOG2;
2795                 else if (strcmp(modifier, "usecs") == 0)
2796                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2797                 else {
2798                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2799                         field = ERR_PTR(-EINVAL);
2800                         goto out;
2801                 }
2802         }
2803
2804         if (strcmp(field_name, "common_timestamp") == 0) {
2805                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2806                 hist_data->enable_timestamps = true;
2807                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2808                         hist_data->attrs->ts_in_usecs = true;
2809         } else if (strcmp(field_name, "cpu") == 0)
2810                 *flags |= HIST_FIELD_FL_CPU;
2811         else {
2812                 field = trace_find_event_field(file->event_call, field_name);
2813                 if (!field || !field->size) {
2814                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
2815                         field = ERR_PTR(-EINVAL);
2816                         goto out;
2817                 }
2818         }
2819  out:
2820         kfree(str);
2821
2822         return field;
2823 }
2824
2825 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2826                                        struct hist_field *var_ref,
2827                                        char *var_name)
2828 {
2829         struct hist_field *alias = NULL;
2830         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2831
2832         alias = create_hist_field(hist_data, NULL, flags, var_name);
2833         if (!alias)
2834                 return NULL;
2835
2836         alias->fn = var_ref->fn;
2837         alias->operands[0] = var_ref;
2838
2839         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2840                 destroy_hist_field(alias, 0);
2841                 return NULL;
2842         }
2843
2844         alias->var_ref_idx = var_ref->var_ref_idx;
2845
2846         return alias;
2847 }
2848
2849 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2850                                      struct trace_event_file *file, char *str,
2851                                      unsigned long *flags, char *var_name)
2852 {
2853         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2854         struct ftrace_event_field *field = NULL;
2855         struct hist_field *hist_field = NULL;
2856         int ret = 0;
2857
2858         s = strchr(str, '.');
2859         if (s) {
2860                 s = strchr(++s, '.');
2861                 if (s) {
2862                         ref_system = strsep(&str, ".");
2863                         if (!str) {
2864                                 ret = -EINVAL;
2865                                 goto out;
2866                         }
2867                         ref_event = strsep(&str, ".");
2868                         if (!str) {
2869                                 ret = -EINVAL;
2870                                 goto out;
2871                         }
2872                         ref_var = str;
2873                 }
2874         }
2875
2876         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2877         if (!s) {
2878                 hist_field = parse_var_ref(hist_data, ref_system,
2879                                            ref_event, ref_var);
2880                 if (hist_field) {
2881                         if (var_name) {
2882                                 hist_field = create_alias(hist_data, hist_field, var_name);
2883                                 if (!hist_field) {
2884                                         ret = -ENOMEM;
2885                                         goto out;
2886                                 }
2887                         }
2888                         return hist_field;
2889                 }
2890         } else
2891                 str = s;
2892
2893         field = parse_field(hist_data, file, str, flags);
2894         if (IS_ERR(field)) {
2895                 ret = PTR_ERR(field);
2896                 goto out;
2897         }
2898
2899         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2900         if (!hist_field) {
2901                 ret = -ENOMEM;
2902                 goto out;
2903         }
2904
2905         return hist_field;
2906  out:
2907         return ERR_PTR(ret);
2908 }
2909
2910 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2911                                      struct trace_event_file *file,
2912                                      char *str, unsigned long flags,
2913                                      char *var_name, unsigned int level);
2914
2915 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2916                                       struct trace_event_file *file,
2917                                       char *str, unsigned long flags,
2918                                       char *var_name, unsigned int level)
2919 {
2920         struct hist_field *operand1, *expr = NULL;
2921         unsigned long operand_flags;
2922         int ret = 0;
2923         char *s;
2924
2925         /* we support only -(xxx) i.e. explicit parens required */
2926
2927         if (level > 3) {
2928                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2929                 ret = -EINVAL;
2930                 goto free;
2931         }
2932
2933         str++; /* skip leading '-' */
2934
2935         s = strchr(str, '(');
2936         if (s)
2937                 str++;
2938         else {
2939                 ret = -EINVAL;
2940                 goto free;
2941         }
2942
2943         s = strrchr(str, ')');
2944         if (s)
2945                 *s = '\0';
2946         else {
2947                 ret = -EINVAL; /* no closing ')' */
2948                 goto free;
2949         }
2950
2951         flags |= HIST_FIELD_FL_EXPR;
2952         expr = create_hist_field(hist_data, NULL, flags, var_name);
2953         if (!expr) {
2954                 ret = -ENOMEM;
2955                 goto free;
2956         }
2957
2958         operand_flags = 0;
2959         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2960         if (IS_ERR(operand1)) {
2961                 ret = PTR_ERR(operand1);
2962                 goto free;
2963         }
2964
2965         expr->flags |= operand1->flags &
2966                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2967         expr->fn = hist_field_unary_minus;
2968         expr->operands[0] = operand1;
2969         expr->operator = FIELD_OP_UNARY_MINUS;
2970         expr->name = expr_str(expr, 0);
2971         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2972         if (!expr->type) {
2973                 ret = -ENOMEM;
2974                 goto free;
2975         }
2976
2977         return expr;
2978  free:
2979         destroy_hist_field(expr, 0);
2980         return ERR_PTR(ret);
2981 }
2982
2983 static int check_expr_operands(struct trace_array *tr,
2984                                struct hist_field *operand1,
2985                                struct hist_field *operand2)
2986 {
2987         unsigned long operand1_flags = operand1->flags;
2988         unsigned long operand2_flags = operand2->flags;
2989
2990         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2991             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2992                 struct hist_field *var;
2993
2994                 var = find_var_field(operand1->var.hist_data, operand1->name);
2995                 if (!var)
2996                         return -EINVAL;
2997                 operand1_flags = var->flags;
2998         }
2999
3000         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
3001             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
3002                 struct hist_field *var;
3003
3004                 var = find_var_field(operand2->var.hist_data, operand2->name);
3005                 if (!var)
3006                         return -EINVAL;
3007                 operand2_flags = var->flags;
3008         }
3009
3010         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
3011             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
3012                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
3013                 return -EINVAL;
3014         }
3015
3016         return 0;
3017 }
3018
3019 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
3020                                      struct trace_event_file *file,
3021                                      char *str, unsigned long flags,
3022                                      char *var_name, unsigned int level)
3023 {
3024         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
3025         unsigned long operand_flags;
3026         int field_op, ret = -EINVAL;
3027         char *sep, *operand1_str;
3028
3029         if (level > 3) {
3030                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
3031                 return ERR_PTR(-EINVAL);
3032         }
3033
3034         field_op = contains_operator(str);
3035
3036         if (field_op == FIELD_OP_NONE)
3037                 return parse_atom(hist_data, file, str, &flags, var_name);
3038
3039         if (field_op == FIELD_OP_UNARY_MINUS)
3040                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
3041
3042         switch (field_op) {
3043         case FIELD_OP_MINUS:
3044                 sep = "-";
3045                 break;
3046         case FIELD_OP_PLUS:
3047                 sep = "+";
3048                 break;
3049         default:
3050                 goto free;
3051         }
3052
3053         operand1_str = strsep(&str, sep);
3054         if (!operand1_str || !str)
3055                 goto free;
3056
3057         operand_flags = 0;
3058         operand1 = parse_atom(hist_data, file, operand1_str,
3059                               &operand_flags, NULL);
3060         if (IS_ERR(operand1)) {
3061                 ret = PTR_ERR(operand1);
3062                 operand1 = NULL;
3063                 goto free;
3064         }
3065
3066         /* rest of string could be another expression e.g. b+c in a+b+c */
3067         operand_flags = 0;
3068         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3069         if (IS_ERR(operand2)) {
3070                 ret = PTR_ERR(operand2);
3071                 operand2 = NULL;
3072                 goto free;
3073         }
3074
3075         ret = check_expr_operands(file->tr, operand1, operand2);
3076         if (ret)
3077                 goto free;
3078
3079         flags |= HIST_FIELD_FL_EXPR;
3080
3081         flags |= operand1->flags &
3082                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3083
3084         expr = create_hist_field(hist_data, NULL, flags, var_name);
3085         if (!expr) {
3086                 ret = -ENOMEM;
3087                 goto free;
3088         }
3089
3090         operand1->read_once = true;
3091         operand2->read_once = true;
3092
3093         expr->operands[0] = operand1;
3094         expr->operands[1] = operand2;
3095         expr->operator = field_op;
3096         expr->name = expr_str(expr, 0);
3097         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3098         if (!expr->type) {
3099                 ret = -ENOMEM;
3100                 goto free;
3101         }
3102
3103         switch (field_op) {
3104         case FIELD_OP_MINUS:
3105                 expr->fn = hist_field_minus;
3106                 break;
3107         case FIELD_OP_PLUS:
3108                 expr->fn = hist_field_plus;
3109                 break;
3110         default:
3111                 ret = -EINVAL;
3112                 goto free;
3113         }
3114
3115         return expr;
3116  free:
3117         destroy_hist_field(operand1, 0);
3118         destroy_hist_field(operand2, 0);
3119         destroy_hist_field(expr, 0);
3120
3121         return ERR_PTR(ret);
3122 }
3123
3124 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3125                                  struct trace_event_file *file)
3126 {
3127         struct event_trigger_data *test;
3128
3129         lockdep_assert_held(&event_mutex);
3130
3131         list_for_each_entry(test, &file->triggers, list) {
3132                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3133                         if (test->private_data == hist_data)
3134                                 return test->filter_str;
3135                 }
3136         }
3137
3138         return NULL;
3139 }
3140
3141 static struct event_command trigger_hist_cmd;
3142 static int event_hist_trigger_func(struct event_command *cmd_ops,
3143                                    struct trace_event_file *file,
3144                                    char *glob, char *cmd, char *param);
3145
3146 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3147                             struct hist_trigger_data *hist_data,
3148                             unsigned int n_keys)
3149 {
3150         struct hist_field *target_hist_field, *hist_field;
3151         unsigned int n, i, j;
3152
3153         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3154                 return false;
3155
3156         i = hist_data->n_vals;
3157         j = target_hist_data->n_vals;
3158
3159         for (n = 0; n < n_keys; n++) {
3160                 hist_field = hist_data->fields[i + n];
3161                 target_hist_field = target_hist_data->fields[j + n];
3162
3163                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3164                         return false;
3165                 if (hist_field->size != target_hist_field->size)
3166                         return false;
3167                 if (hist_field->is_signed != target_hist_field->is_signed)
3168                         return false;
3169         }
3170
3171         return true;
3172 }
3173
3174 static struct hist_trigger_data *
3175 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3176                      struct trace_event_file *file)
3177 {
3178         struct hist_trigger_data *hist_data;
3179         struct event_trigger_data *test;
3180         unsigned int n_keys;
3181
3182         lockdep_assert_held(&event_mutex);
3183
3184         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3185
3186         list_for_each_entry(test, &file->triggers, list) {
3187                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3188                         hist_data = test->private_data;
3189
3190                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3191                                 return hist_data;
3192                 }
3193         }
3194
3195         return NULL;
3196 }
3197
3198 static struct trace_event_file *event_file(struct trace_array *tr,
3199                                            char *system, char *event_name)
3200 {
3201         struct trace_event_file *file;
3202
3203         file = __find_event_file(tr, system, event_name);
3204         if (!file)
3205                 return ERR_PTR(-EINVAL);
3206
3207         return file;
3208 }
3209
3210 static struct hist_field *
3211 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3212                          char *system, char *event_name, char *field_name)
3213 {
3214         struct hist_field *event_var;
3215         char *synthetic_name;
3216
3217         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3218         if (!synthetic_name)
3219                 return ERR_PTR(-ENOMEM);
3220
3221         strcpy(synthetic_name, "synthetic_");
3222         strcat(synthetic_name, field_name);
3223
3224         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3225
3226         kfree(synthetic_name);
3227
3228         return event_var;
3229 }
3230
3231 /**
3232  * create_field_var_hist - Automatically create a histogram and var for a field
3233  * @target_hist_data: The target hist trigger
3234  * @subsys_name: Optional subsystem name
3235  * @event_name: Optional event name
3236  * @field_name: The name of the field (and the resulting variable)
3237  *
3238  * Hist trigger actions fetch data from variables, not directly from
3239  * events.  However, for convenience, users are allowed to directly
3240  * specify an event field in an action, which will be automatically
3241  * converted into a variable on their behalf.
3242
3243  * If a user specifies a field on an event that isn't the event the
3244  * histogram currently being defined (the target event histogram), the
3245  * only way that can be accomplished is if a new hist trigger is
3246  * created and the field variable defined on that.
3247  *
3248  * This function creates a new histogram compatible with the target
3249  * event (meaning a histogram with the same key as the target
3250  * histogram), and creates a variable for the specified field, but
3251  * with 'synthetic_' prepended to the variable name in order to avoid
3252  * collision with normal field variables.
3253  *
3254  * Return: The variable created for the field.
3255  */
3256 static struct hist_field *
3257 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3258                       char *subsys_name, char *event_name, char *field_name)
3259 {
3260         struct trace_array *tr = target_hist_data->event_file->tr;
3261         struct hist_field *event_var = ERR_PTR(-EINVAL);
3262         struct hist_trigger_data *hist_data;
3263         unsigned int i, n, first = true;
3264         struct field_var_hist *var_hist;
3265         struct trace_event_file *file;
3266         struct hist_field *key_field;
3267         char *saved_filter;
3268         char *cmd;
3269         int ret;
3270
3271         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3272                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3273                 return ERR_PTR(-EINVAL);
3274         }
3275
3276         file = event_file(tr, subsys_name, event_name);
3277
3278         if (IS_ERR(file)) {
3279                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3280                 ret = PTR_ERR(file);
3281                 return ERR_PTR(ret);
3282         }
3283
3284         /*
3285          * Look for a histogram compatible with target.  We'll use the
3286          * found histogram specification to create a new matching
3287          * histogram with our variable on it.  target_hist_data is not
3288          * yet a registered histogram so we can't use that.
3289          */
3290         hist_data = find_compatible_hist(target_hist_data, file);
3291         if (!hist_data) {
3292                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3293                 return ERR_PTR(-EINVAL);
3294         }
3295
3296         /* See if a synthetic field variable has already been created */
3297         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3298                                              event_name, field_name);
3299         if (!IS_ERR_OR_NULL(event_var))
3300                 return event_var;
3301
3302         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3303         if (!var_hist)
3304                 return ERR_PTR(-ENOMEM);
3305
3306         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3307         if (!cmd) {
3308                 kfree(var_hist);
3309                 return ERR_PTR(-ENOMEM);
3310         }
3311
3312         /* Use the same keys as the compatible histogram */
3313         strcat(cmd, "keys=");
3314
3315         for_each_hist_key_field(i, hist_data) {
3316                 key_field = hist_data->fields[i];
3317                 if (!first)
3318                         strcat(cmd, ",");
3319                 strcat(cmd, key_field->field->name);
3320                 first = false;
3321         }
3322
3323         /* Create the synthetic field variable specification */
3324         strcat(cmd, ":synthetic_");
3325         strcat(cmd, field_name);
3326         strcat(cmd, "=");
3327         strcat(cmd, field_name);
3328
3329         /* Use the same filter as the compatible histogram */
3330         saved_filter = find_trigger_filter(hist_data, file);
3331         if (saved_filter) {
3332                 strcat(cmd, " if ");
3333                 strcat(cmd, saved_filter);
3334         }
3335
3336         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3337         if (!var_hist->cmd) {
3338                 kfree(cmd);
3339                 kfree(var_hist);
3340                 return ERR_PTR(-ENOMEM);
3341         }
3342
3343         /* Save the compatible histogram information */
3344         var_hist->hist_data = hist_data;
3345
3346         /* Create the new histogram with our variable */
3347         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3348                                       "", "hist", cmd);
3349         if (ret) {
3350                 kfree(cmd);
3351                 kfree(var_hist->cmd);
3352                 kfree(var_hist);
3353                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3354                 return ERR_PTR(ret);
3355         }
3356
3357         kfree(cmd);
3358
3359         /* If we can't find the variable, something went wrong */
3360         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3361                                              event_name, field_name);
3362         if (IS_ERR_OR_NULL(event_var)) {
3363                 kfree(var_hist->cmd);
3364                 kfree(var_hist);
3365                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3366                 return ERR_PTR(-EINVAL);
3367         }
3368
3369         n = target_hist_data->n_field_var_hists;
3370         target_hist_data->field_var_hists[n] = var_hist;
3371         target_hist_data->n_field_var_hists++;
3372
3373         return event_var;
3374 }
3375
3376 static struct hist_field *
3377 find_target_event_var(struct hist_trigger_data *hist_data,
3378                       char *subsys_name, char *event_name, char *var_name)
3379 {
3380         struct trace_event_file *file = hist_data->event_file;
3381         struct hist_field *hist_field = NULL;
3382
3383         if (subsys_name) {
3384                 struct trace_event_call *call;
3385
3386                 if (!event_name)
3387                         return NULL;
3388
3389                 call = file->event_call;
3390
3391                 if (strcmp(subsys_name, call->class->system) != 0)
3392                         return NULL;
3393
3394                 if (strcmp(event_name, trace_event_name(call)) != 0)
3395                         return NULL;
3396         }
3397
3398         hist_field = find_var_field(hist_data, var_name);
3399
3400         return hist_field;
3401 }
3402
3403 static inline void __update_field_vars(struct tracing_map_elt *elt,
3404                                        struct ring_buffer_event *rbe,
3405                                        void *rec,
3406                                        struct field_var **field_vars,
3407                                        unsigned int n_field_vars,
3408                                        unsigned int field_var_str_start)
3409 {
3410         struct hist_elt_data *elt_data = elt->private_data;
3411         unsigned int i, j, var_idx;
3412         u64 var_val;
3413
3414         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3415                 struct field_var *field_var = field_vars[i];
3416                 struct hist_field *var = field_var->var;
3417                 struct hist_field *val = field_var->val;
3418
3419                 var_val = val->fn(val, elt, rbe, rec);
3420                 var_idx = var->var.idx;
3421
3422                 if (val->flags & HIST_FIELD_FL_STRING) {
3423                         char *str = elt_data->field_var_str[j++];
3424                         char *val_str = (char *)(uintptr_t)var_val;
3425
3426                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3427                         var_val = (u64)(uintptr_t)str;
3428                 }
3429                 tracing_map_set_var(elt, var_idx, var_val);
3430         }
3431 }
3432
3433 static void update_field_vars(struct hist_trigger_data *hist_data,
3434                               struct tracing_map_elt *elt,
3435                               struct ring_buffer_event *rbe,
3436                               void *rec)
3437 {
3438         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3439                             hist_data->n_field_vars, 0);
3440 }
3441
3442 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3443                                  struct tracing_map_elt *elt, void *rec,
3444                                  struct ring_buffer_event *rbe, void *key,
3445                                  struct action_data *data, u64 *var_ref_vals)
3446 {
3447         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
3448                             hist_data->n_save_vars, hist_data->n_field_var_str);
3449 }
3450
3451 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3452                                      struct trace_event_file *file,
3453                                      char *name, int size, const char *type)
3454 {
3455         struct hist_field *var;
3456         int idx;
3457
3458         if (find_var(hist_data, file, name) && !hist_data->remove) {
3459                 var = ERR_PTR(-EINVAL);
3460                 goto out;
3461         }
3462
3463         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3464         if (!var) {
3465                 var = ERR_PTR(-ENOMEM);
3466                 goto out;
3467         }
3468
3469         idx = tracing_map_add_var(hist_data->map);
3470         if (idx < 0) {
3471                 kfree(var);
3472                 var = ERR_PTR(-EINVAL);
3473                 goto out;
3474         }
3475
3476         var->flags = HIST_FIELD_FL_VAR;
3477         var->var.idx = idx;
3478         var->var.hist_data = var->hist_data = hist_data;
3479         var->size = size;
3480         var->var.name = kstrdup(name, GFP_KERNEL);
3481         var->type = kstrdup(type, GFP_KERNEL);
3482         if (!var->var.name || !var->type) {
3483                 kfree(var->var.name);
3484                 kfree(var->type);
3485                 kfree(var);
3486                 var = ERR_PTR(-ENOMEM);
3487         }
3488  out:
3489         return var;
3490 }
3491
3492 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3493                                           struct trace_event_file *file,
3494                                           char *field_name)
3495 {
3496         struct hist_field *val = NULL, *var = NULL;
3497         unsigned long flags = HIST_FIELD_FL_VAR;
3498         struct trace_array *tr = file->tr;
3499         struct field_var *field_var;
3500         int ret = 0;
3501
3502         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3503                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3504                 ret = -EINVAL;
3505                 goto err;
3506         }
3507
3508         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3509         if (IS_ERR(val)) {
3510                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3511                 ret = PTR_ERR(val);
3512                 goto err;
3513         }
3514
3515         var = create_var(hist_data, file, field_name, val->size, val->type);
3516         if (IS_ERR(var)) {
3517                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3518                 kfree(val);
3519                 ret = PTR_ERR(var);
3520                 goto err;
3521         }
3522
3523         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3524         if (!field_var) {
3525                 kfree(val);
3526                 kfree(var);
3527                 ret =  -ENOMEM;
3528                 goto err;
3529         }
3530
3531         field_var->var = var;
3532         field_var->val = val;
3533  out:
3534         return field_var;
3535  err:
3536         field_var = ERR_PTR(ret);
3537         goto out;
3538 }
3539
3540 /**
3541  * create_target_field_var - Automatically create a variable for a field
3542  * @target_hist_data: The target hist trigger
3543  * @subsys_name: Optional subsystem name
3544  * @event_name: Optional event name
3545  * @var_name: The name of the field (and the resulting variable)
3546  *
3547  * Hist trigger actions fetch data from variables, not directly from
3548  * events.  However, for convenience, users are allowed to directly
3549  * specify an event field in an action, which will be automatically
3550  * converted into a variable on their behalf.
3551
3552  * This function creates a field variable with the name var_name on
3553  * the hist trigger currently being defined on the target event.  If
3554  * subsys_name and event_name are specified, this function simply
3555  * verifies that they do in fact match the target event subsystem and
3556  * event name.
3557  *
3558  * Return: The variable created for the field.
3559  */
3560 static struct field_var *
3561 create_target_field_var(struct hist_trigger_data *target_hist_data,
3562                         char *subsys_name, char *event_name, char *var_name)
3563 {
3564         struct trace_event_file *file = target_hist_data->event_file;
3565
3566         if (subsys_name) {
3567                 struct trace_event_call *call;
3568
3569                 if (!event_name)
3570                         return NULL;
3571
3572                 call = file->event_call;
3573
3574                 if (strcmp(subsys_name, call->class->system) != 0)
3575                         return NULL;
3576
3577                 if (strcmp(event_name, trace_event_name(call)) != 0)
3578                         return NULL;
3579         }
3580
3581         return create_field_var(target_hist_data, file, var_name);
3582 }
3583
3584 static bool check_track_val_max(u64 track_val, u64 var_val)
3585 {
3586         if (var_val <= track_val)
3587                 return false;
3588
3589         return true;
3590 }
3591
3592 static bool check_track_val_changed(u64 track_val, u64 var_val)
3593 {
3594         if (var_val == track_val)
3595                 return false;
3596
3597         return true;
3598 }
3599
3600 static u64 get_track_val(struct hist_trigger_data *hist_data,
3601                          struct tracing_map_elt *elt,
3602                          struct action_data *data)
3603 {
3604         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3605         u64 track_val;
3606
3607         track_val = tracing_map_read_var(elt, track_var_idx);
3608
3609         return track_val;
3610 }
3611
3612 static void save_track_val(struct hist_trigger_data *hist_data,
3613                            struct tracing_map_elt *elt,
3614                            struct action_data *data, u64 var_val)
3615 {
3616         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3617
3618         tracing_map_set_var(elt, track_var_idx, var_val);
3619 }
3620
3621 static void save_track_data(struct hist_trigger_data *hist_data,
3622                             struct tracing_map_elt *elt, void *rec,
3623                             struct ring_buffer_event *rbe, void *key,
3624                             struct action_data *data, u64 *var_ref_vals)
3625 {
3626         if (data->track_data.save_data)
3627                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3628 }
3629
3630 static bool check_track_val(struct tracing_map_elt *elt,
3631                             struct action_data *data,
3632                             u64 var_val)
3633 {
3634         struct hist_trigger_data *hist_data;
3635         u64 track_val;
3636
3637         hist_data = data->track_data.track_var->hist_data;
3638         track_val = get_track_val(hist_data, elt, data);
3639
3640         return data->track_data.check_val(track_val, var_val);
3641 }
3642
3643 #ifdef CONFIG_TRACER_SNAPSHOT
3644 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3645 {
3646         /* called with tr->max_lock held */
3647         struct track_data *track_data = tr->cond_snapshot->cond_data;
3648         struct hist_elt_data *elt_data, *track_elt_data;
3649         struct snapshot_context *context = cond_data;
3650         struct action_data *action;
3651         u64 track_val;
3652
3653         if (!track_data)
3654                 return false;
3655
3656         action = track_data->action_data;
3657
3658         track_val = get_track_val(track_data->hist_data, context->elt,
3659                                   track_data->action_data);
3660
3661         if (!action->track_data.check_val(track_data->track_val, track_val))
3662                 return false;
3663
3664         track_data->track_val = track_val;
3665         memcpy(track_data->key, context->key, track_data->key_len);
3666
3667         elt_data = context->elt->private_data;
3668         track_elt_data = track_data->elt.private_data;
3669         if (elt_data->comm)
3670                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3671
3672         track_data->updated = true;
3673
3674         return true;
3675 }
3676
3677 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3678                                      struct tracing_map_elt *elt, void *rec,
3679                                      struct ring_buffer_event *rbe, void *key,
3680                                      struct action_data *data,
3681                                      u64 *var_ref_vals)
3682 {
3683         struct trace_event_file *file = hist_data->event_file;
3684         struct snapshot_context context;
3685
3686         context.elt = elt;
3687         context.key = key;
3688
3689         tracing_snapshot_cond(file->tr, &context);
3690 }
3691
3692 static void hist_trigger_print_key(struct seq_file *m,
3693                                    struct hist_trigger_data *hist_data,
3694                                    void *key,
3695                                    struct tracing_map_elt *elt);
3696
3697 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3698 {
3699         unsigned int i;
3700
3701         if (!hist_data->n_actions)
3702                 return NULL;
3703
3704         for (i = 0; i < hist_data->n_actions; i++) {
3705                 struct action_data *data = hist_data->actions[i];
3706
3707                 if (data->action == ACTION_SNAPSHOT)
3708                         return data;
3709         }
3710
3711         return NULL;
3712 }
3713
3714 static void track_data_snapshot_print(struct seq_file *m,
3715                                       struct hist_trigger_data *hist_data)
3716 {
3717         struct trace_event_file *file = hist_data->event_file;
3718         struct track_data *track_data;
3719         struct action_data *action;
3720
3721         track_data = tracing_cond_snapshot_data(file->tr);
3722         if (!track_data)
3723                 return;
3724
3725         if (!track_data->updated)
3726                 return;
3727
3728         action = snapshot_action(hist_data);
3729         if (!action)
3730                 return;
3731
3732         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3733         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3734                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3735                    action->track_data.var_str, track_data->track_val);
3736
3737         seq_puts(m, "\ttriggered by event with key: ");
3738         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3739         seq_putc(m, '\n');
3740 }
3741 #else
3742 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3743 {
3744         return false;
3745 }
3746 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3747                                      struct tracing_map_elt *elt, void *rec,
3748                                      struct ring_buffer_event *rbe, void *key,
3749                                      struct action_data *data,
3750                                      u64 *var_ref_vals) {}
3751 static void track_data_snapshot_print(struct seq_file *m,
3752                                       struct hist_trigger_data *hist_data) {}
3753 #endif /* CONFIG_TRACER_SNAPSHOT */
3754
3755 static void track_data_print(struct seq_file *m,
3756                              struct hist_trigger_data *hist_data,
3757                              struct tracing_map_elt *elt,
3758                              struct action_data *data)
3759 {
3760         u64 track_val = get_track_val(hist_data, elt, data);
3761         unsigned int i, save_var_idx;
3762
3763         if (data->handler == HANDLER_ONMAX)
3764                 seq_printf(m, "\n\tmax: %10llu", track_val);
3765         else if (data->handler == HANDLER_ONCHANGE)
3766                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3767
3768         if (data->action == ACTION_SNAPSHOT)
3769                 return;
3770
3771         for (i = 0; i < hist_data->n_save_vars; i++) {
3772                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3773                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3774                 u64 val;
3775
3776                 save_var_idx = save_var->var.idx;
3777
3778                 val = tracing_map_read_var(elt, save_var_idx);
3779
3780                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3781                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3782                                    (char *)(uintptr_t)(val));
3783                 } else
3784                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3785         }
3786 }
3787
3788 static void ontrack_action(struct hist_trigger_data *hist_data,
3789                            struct tracing_map_elt *elt, void *rec,
3790                            struct ring_buffer_event *rbe, void *key,
3791                            struct action_data *data, u64 *var_ref_vals)
3792 {
3793         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3794
3795         if (check_track_val(elt, data, var_val)) {
3796                 save_track_val(hist_data, elt, data, var_val);
3797                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3798         }
3799 }
3800
3801 static void action_data_destroy(struct action_data *data)
3802 {
3803         unsigned int i;
3804
3805         lockdep_assert_held(&event_mutex);
3806
3807         kfree(data->action_name);
3808
3809         for (i = 0; i < data->n_params; i++)
3810                 kfree(data->params[i]);
3811
3812         if (data->synth_event)
3813                 data->synth_event->ref--;
3814
3815         kfree(data->synth_event_name);
3816
3817         kfree(data);
3818 }
3819
3820 static void track_data_destroy(struct hist_trigger_data *hist_data,
3821                                struct action_data *data)
3822 {
3823         struct trace_event_file *file = hist_data->event_file;
3824
3825         destroy_hist_field(data->track_data.track_var, 0);
3826
3827         if (data->action == ACTION_SNAPSHOT) {
3828                 struct track_data *track_data;
3829
3830                 track_data = tracing_cond_snapshot_data(file->tr);
3831                 if (track_data && track_data->hist_data == hist_data) {
3832                         tracing_snapshot_cond_disable(file->tr);
3833                         track_data_free(track_data);
3834                 }
3835         }
3836
3837         kfree(data->track_data.var_str);
3838
3839         action_data_destroy(data);
3840 }
3841
3842 static int action_create(struct hist_trigger_data *hist_data,
3843                          struct action_data *data);
3844
3845 static int track_data_create(struct hist_trigger_data *hist_data,
3846                              struct action_data *data)
3847 {
3848         struct hist_field *var_field, *ref_field, *track_var = NULL;
3849         struct trace_event_file *file = hist_data->event_file;
3850         struct trace_array *tr = file->tr;
3851         char *track_data_var_str;
3852         int ret = 0;
3853
3854         track_data_var_str = data->track_data.var_str;
3855         if (track_data_var_str[0] != '$') {
3856                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3857                 return -EINVAL;
3858         }
3859         track_data_var_str++;
3860
3861         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3862         if (!var_field) {
3863                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3864                 return -EINVAL;
3865         }
3866
3867         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3868         if (!ref_field)
3869                 return -ENOMEM;
3870
3871         data->track_data.var_ref = ref_field;
3872
3873         if (data->handler == HANDLER_ONMAX)
3874                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3875         if (IS_ERR(track_var)) {
3876                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3877                 ret = PTR_ERR(track_var);
3878                 goto out;
3879         }
3880
3881         if (data->handler == HANDLER_ONCHANGE)
3882                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3883         if (IS_ERR(track_var)) {
3884                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3885                 ret = PTR_ERR(track_var);
3886                 goto out;
3887         }
3888         data->track_data.track_var = track_var;
3889
3890         ret = action_create(hist_data, data);
3891  out:
3892         return ret;
3893 }
3894
3895 static int parse_action_params(struct trace_array *tr, char *params,
3896                                struct action_data *data)
3897 {
3898         char *param, *saved_param;
3899         bool first_param = true;
3900         int ret = 0;
3901
3902         while (params) {
3903                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3904                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3905                         goto out;
3906                 }
3907
3908                 param = strsep(&params, ",");
3909                 if (!param) {
3910                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3911                         ret = -EINVAL;
3912                         goto out;
3913                 }
3914
3915                 param = strstrip(param);
3916                 if (strlen(param) < 2) {
3917                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3918                         ret = -EINVAL;
3919                         goto out;
3920                 }
3921
3922                 saved_param = kstrdup(param, GFP_KERNEL);
3923                 if (!saved_param) {
3924                         ret = -ENOMEM;
3925                         goto out;
3926                 }
3927
3928                 if (first_param && data->use_trace_keyword) {
3929                         data->synth_event_name = saved_param;
3930                         first_param = false;
3931                         continue;
3932                 }
3933                 first_param = false;
3934
3935                 data->params[data->n_params++] = saved_param;
3936         }
3937  out:
3938         return ret;
3939 }
3940
3941 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3942                         enum handler_id handler)
3943 {
3944         char *action_name;
3945         int ret = 0;
3946
3947         strsep(&str, ".");
3948         if (!str) {
3949                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3950                 ret = -EINVAL;
3951                 goto out;
3952         }
3953
3954         action_name = strsep(&str, "(");
3955         if (!action_name || !str) {
3956                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3957                 ret = -EINVAL;
3958                 goto out;
3959         }
3960
3961         if (str_has_prefix(action_name, "save")) {
3962                 char *params = strsep(&str, ")");
3963
3964                 if (!params) {
3965                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3966                         ret = -EINVAL;
3967                         goto out;
3968                 }
3969
3970                 ret = parse_action_params(tr, params, data);
3971                 if (ret)
3972                         goto out;
3973
3974                 if (handler == HANDLER_ONMAX)
3975                         data->track_data.check_val = check_track_val_max;
3976                 else if (handler == HANDLER_ONCHANGE)
3977                         data->track_data.check_val = check_track_val_changed;
3978                 else {
3979                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3980                         ret = -EINVAL;
3981                         goto out;
3982                 }
3983
3984                 data->track_data.save_data = save_track_data_vars;
3985                 data->fn = ontrack_action;
3986                 data->action = ACTION_SAVE;
3987         } else if (str_has_prefix(action_name, "snapshot")) {
3988                 char *params = strsep(&str, ")");
3989
3990                 if (!str) {
3991                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3992                         ret = -EINVAL;
3993                         goto out;
3994                 }
3995
3996                 if (handler == HANDLER_ONMAX)
3997                         data->track_data.check_val = check_track_val_max;
3998                 else if (handler == HANDLER_ONCHANGE)
3999                         data->track_data.check_val = check_track_val_changed;
4000                 else {
4001                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
4002                         ret = -EINVAL;
4003                         goto out;
4004                 }
4005
4006                 data->track_data.save_data = save_track_data_snapshot;
4007                 data->fn = ontrack_action;
4008                 data->action = ACTION_SNAPSHOT;
4009         } else {
4010                 char *params = strsep(&str, ")");
4011
4012                 if (str_has_prefix(action_name, "trace"))
4013                         data->use_trace_keyword = true;
4014
4015                 if (params) {
4016                         ret = parse_action_params(tr, params, data);
4017                         if (ret)
4018                                 goto out;
4019                 }
4020
4021                 if (handler == HANDLER_ONMAX)
4022                         data->track_data.check_val = check_track_val_max;
4023                 else if (handler == HANDLER_ONCHANGE)
4024                         data->track_data.check_val = check_track_val_changed;
4025
4026                 if (handler != HANDLER_ONMATCH) {
4027                         data->track_data.save_data = action_trace;
4028                         data->fn = ontrack_action;
4029                 } else
4030                         data->fn = action_trace;
4031
4032                 data->action = ACTION_TRACE;
4033         }
4034
4035         data->action_name = kstrdup(action_name, GFP_KERNEL);
4036         if (!data->action_name) {
4037                 ret = -ENOMEM;
4038                 goto out;
4039         }
4040
4041         data->handler = handler;
4042  out:
4043         return ret;
4044 }
4045
4046 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
4047                                             char *str, enum handler_id handler)
4048 {
4049         struct action_data *data;
4050         int ret = -EINVAL;
4051         char *var_str;
4052
4053         data = kzalloc(sizeof(*data), GFP_KERNEL);
4054         if (!data)
4055                 return ERR_PTR(-ENOMEM);
4056
4057         var_str = strsep(&str, ")");
4058         if (!var_str || !str) {
4059                 ret = -EINVAL;
4060                 goto free;
4061         }
4062
4063         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
4064         if (!data->track_data.var_str) {
4065                 ret = -ENOMEM;
4066                 goto free;
4067         }
4068
4069         ret = action_parse(hist_data->event_file->tr, str, data, handler);
4070         if (ret)
4071                 goto free;
4072  out:
4073         return data;
4074  free:
4075         track_data_destroy(hist_data, data);
4076         data = ERR_PTR(ret);
4077         goto out;
4078 }
4079
4080 static void onmatch_destroy(struct action_data *data)
4081 {
4082         kfree(data->match_data.event);
4083         kfree(data->match_data.event_system);
4084
4085         action_data_destroy(data);
4086 }
4087
4088 static void destroy_field_var(struct field_var *field_var)
4089 {
4090         if (!field_var)
4091                 return;
4092
4093         destroy_hist_field(field_var->var, 0);
4094         destroy_hist_field(field_var->val, 0);
4095
4096         kfree(field_var);
4097 }
4098
4099 static void destroy_field_vars(struct hist_trigger_data *hist_data)
4100 {
4101         unsigned int i;
4102
4103         for (i = 0; i < hist_data->n_field_vars; i++)
4104                 destroy_field_var(hist_data->field_vars[i]);
4105 }
4106
4107 static void save_field_var(struct hist_trigger_data *hist_data,
4108                            struct field_var *field_var)
4109 {
4110         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
4111
4112         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4113                 hist_data->n_field_var_str++;
4114 }
4115
4116
4117 static int check_synth_field(struct synth_event *event,
4118                              struct hist_field *hist_field,
4119                              unsigned int field_pos)
4120 {
4121         struct synth_field *field;
4122
4123         if (field_pos >= event->n_fields)
4124                 return -EINVAL;
4125
4126         field = event->fields[field_pos];
4127
4128         if (strcmp(field->type, hist_field->type) != 0) {
4129                 if (field->size != hist_field->size ||
4130                     field->is_signed != hist_field->is_signed)
4131                         return -EINVAL;
4132         }
4133
4134         return 0;
4135 }
4136
4137 static struct hist_field *
4138 trace_action_find_var(struct hist_trigger_data *hist_data,
4139                       struct action_data *data,
4140                       char *system, char *event, char *var)
4141 {
4142         struct trace_array *tr = hist_data->event_file->tr;
4143         struct hist_field *hist_field;
4144
4145         var++; /* skip '$' */
4146
4147         hist_field = find_target_event_var(hist_data, system, event, var);
4148         if (!hist_field) {
4149                 if (!system && data->handler == HANDLER_ONMATCH) {
4150                         system = data->match_data.event_system;
4151                         event = data->match_data.event;
4152                 }
4153
4154                 hist_field = find_event_var(hist_data, system, event, var);
4155         }
4156
4157         if (!hist_field)
4158                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
4159
4160         return hist_field;
4161 }
4162
4163 static struct hist_field *
4164 trace_action_create_field_var(struct hist_trigger_data *hist_data,
4165                               struct action_data *data, char *system,
4166                               char *event, char *var)
4167 {
4168         struct hist_field *hist_field = NULL;
4169         struct field_var *field_var;
4170
4171         /*
4172          * First try to create a field var on the target event (the
4173          * currently being defined).  This will create a variable for
4174          * unqualified fields on the target event, or if qualified,
4175          * target fields that have qualified names matching the target.
4176          */
4177         field_var = create_target_field_var(hist_data, system, event, var);
4178
4179         if (field_var && !IS_ERR(field_var)) {
4180                 save_field_var(hist_data, field_var);
4181                 hist_field = field_var->var;
4182         } else {
4183                 field_var = NULL;
4184                 /*
4185                  * If no explicit system.event is specfied, default to
4186                  * looking for fields on the onmatch(system.event.xxx)
4187                  * event.
4188                  */
4189                 if (!system && data->handler == HANDLER_ONMATCH) {
4190                         system = data->match_data.event_system;
4191                         event = data->match_data.event;
4192                 }
4193
4194                 /*
4195                  * At this point, we're looking at a field on another
4196                  * event.  Because we can't modify a hist trigger on
4197                  * another event to add a variable for a field, we need
4198                  * to create a new trigger on that event and create the
4199                  * variable at the same time.
4200                  */
4201                 hist_field = create_field_var_hist(hist_data, system, event, var);
4202                 if (IS_ERR(hist_field))
4203                         goto free;
4204         }
4205  out:
4206         return hist_field;
4207  free:
4208         destroy_field_var(field_var);
4209         hist_field = NULL;
4210         goto out;
4211 }
4212
4213 static int trace_action_create(struct hist_trigger_data *hist_data,
4214                                struct action_data *data)
4215 {
4216         struct trace_array *tr = hist_data->event_file->tr;
4217         char *event_name, *param, *system = NULL;
4218         struct hist_field *hist_field, *var_ref;
4219         unsigned int i, var_ref_idx;
4220         unsigned int field_pos = 0;
4221         struct synth_event *event;
4222         char *synth_event_name;
4223         int ret = 0;
4224
4225         lockdep_assert_held(&event_mutex);
4226
4227         if (data->use_trace_keyword)
4228                 synth_event_name = data->synth_event_name;
4229         else
4230                 synth_event_name = data->action_name;
4231
4232         event = find_synth_event(synth_event_name);
4233         if (!event) {
4234                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4235                 return -EINVAL;
4236         }
4237
4238         event->ref++;
4239
4240         var_ref_idx = hist_data->n_var_refs;
4241
4242         for (i = 0; i < data->n_params; i++) {
4243                 char *p;
4244
4245                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4246                 if (!param) {
4247                         ret = -ENOMEM;
4248                         goto err;
4249                 }
4250
4251                 system = strsep(&param, ".");
4252                 if (!param) {
4253                         param = (char *)system;
4254                         system = event_name = NULL;
4255                 } else {
4256                         event_name = strsep(&param, ".");
4257                         if (!param) {
4258                                 kfree(p);
4259                                 ret = -EINVAL;
4260                                 goto err;
4261                         }
4262                 }
4263
4264                 if (param[0] == '$')
4265                         hist_field = trace_action_find_var(hist_data, data,
4266                                                            system, event_name,
4267                                                            param);
4268                 else
4269                         hist_field = trace_action_create_field_var(hist_data,
4270                                                                    data,
4271                                                                    system,
4272                                                                    event_name,
4273                                                                    param);
4274
4275                 if (!hist_field) {
4276                         kfree(p);
4277                         ret = -EINVAL;
4278                         goto err;
4279                 }
4280
4281                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4282                         var_ref = create_var_ref(hist_data, hist_field,
4283                                                  system, event_name);
4284                         if (!var_ref) {
4285                                 kfree(p);
4286                                 ret = -ENOMEM;
4287                                 goto err;
4288                         }
4289
4290                         field_pos++;
4291                         kfree(p);
4292                         continue;
4293                 }
4294
4295                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4296                 kfree(p);
4297                 ret = -EINVAL;
4298                 goto err;
4299         }
4300
4301         if (field_pos != event->n_fields) {
4302                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4303                 ret = -EINVAL;
4304                 goto err;
4305         }
4306
4307         data->synth_event = event;
4308         data->var_ref_idx = var_ref_idx;
4309  out:
4310         return ret;
4311  err:
4312         event->ref--;
4313
4314         goto out;
4315 }
4316
4317 static int action_create(struct hist_trigger_data *hist_data,
4318                          struct action_data *data)
4319 {
4320         struct trace_event_file *file = hist_data->event_file;
4321         struct trace_array *tr = file->tr;
4322         struct track_data *track_data;
4323         struct field_var *field_var;
4324         unsigned int i;
4325         char *param;
4326         int ret = 0;
4327
4328         if (data->action == ACTION_TRACE)
4329                 return trace_action_create(hist_data, data);
4330
4331         if (data->action == ACTION_SNAPSHOT) {
4332                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4333                 if (IS_ERR(track_data)) {
4334                         ret = PTR_ERR(track_data);
4335                         goto out;
4336                 }
4337
4338                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4339                                                    cond_snapshot_update);
4340                 if (ret)
4341                         track_data_free(track_data);
4342
4343                 goto out;
4344         }
4345
4346         if (data->action == ACTION_SAVE) {
4347                 if (hist_data->n_save_vars) {
4348                         ret = -EEXIST;
4349                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4350                         goto out;
4351                 }
4352
4353                 for (i = 0; i < data->n_params; i++) {
4354                         param = kstrdup(data->params[i], GFP_KERNEL);
4355                         if (!param) {
4356                                 ret = -ENOMEM;
4357                                 goto out;
4358                         }
4359
4360                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4361                         if (IS_ERR(field_var)) {
4362                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4363                                          errpos(param));
4364                                 ret = PTR_ERR(field_var);
4365                                 kfree(param);
4366                                 goto out;
4367                         }
4368
4369                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4370                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4371                                 hist_data->n_save_var_str++;
4372                         kfree(param);
4373                 }
4374         }
4375  out:
4376         return ret;
4377 }
4378
4379 static int onmatch_create(struct hist_trigger_data *hist_data,
4380                           struct action_data *data)
4381 {
4382         return action_create(hist_data, data);
4383 }
4384
4385 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4386 {
4387         char *match_event, *match_event_system;
4388         struct action_data *data;
4389         int ret = -EINVAL;
4390
4391         data = kzalloc(sizeof(*data), GFP_KERNEL);
4392         if (!data)
4393                 return ERR_PTR(-ENOMEM);
4394
4395         match_event = strsep(&str, ")");
4396         if (!match_event || !str) {
4397                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4398                 goto free;
4399         }
4400
4401         match_event_system = strsep(&match_event, ".");
4402         if (!match_event) {
4403                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4404                 goto free;
4405         }
4406
4407         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4408                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4409                 goto free;
4410         }
4411
4412         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4413         if (!data->match_data.event) {
4414                 ret = -ENOMEM;
4415                 goto free;
4416         }
4417
4418         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4419         if (!data->match_data.event_system) {
4420                 ret = -ENOMEM;
4421                 goto free;
4422         }
4423
4424         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4425         if (ret)
4426                 goto free;
4427  out:
4428         return data;
4429  free:
4430         onmatch_destroy(data);
4431         data = ERR_PTR(ret);
4432         goto out;
4433 }
4434
4435 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4436 {
4437         hist_data->fields[HITCOUNT_IDX] =
4438                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4439         if (!hist_data->fields[HITCOUNT_IDX])
4440                 return -ENOMEM;
4441
4442         hist_data->n_vals++;
4443         hist_data->n_fields++;
4444
4445         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4446                 return -EINVAL;
4447
4448         return 0;
4449 }
4450
4451 static int __create_val_field(struct hist_trigger_data *hist_data,
4452                               unsigned int val_idx,
4453                               struct trace_event_file *file,
4454                               char *var_name, char *field_str,
4455                               unsigned long flags)
4456 {
4457         struct hist_field *hist_field;
4458         int ret = 0;
4459
4460         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4461         if (IS_ERR(hist_field)) {
4462                 ret = PTR_ERR(hist_field);
4463                 goto out;
4464         }
4465
4466         hist_data->fields[val_idx] = hist_field;
4467
4468         ++hist_data->n_vals;
4469         ++hist_data->n_fields;
4470
4471         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4472                 ret = -EINVAL;
4473  out:
4474         return ret;
4475 }
4476
4477 static int create_val_field(struct hist_trigger_data *hist_data,
4478                             unsigned int val_idx,
4479                             struct trace_event_file *file,
4480                             char *field_str)
4481 {
4482         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4483                 return -EINVAL;
4484
4485         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4486 }
4487
4488 static int create_var_field(struct hist_trigger_data *hist_data,
4489                             unsigned int val_idx,
4490                             struct trace_event_file *file,
4491                             char *var_name, char *expr_str)
4492 {
4493         struct trace_array *tr = hist_data->event_file->tr;
4494         unsigned long flags = 0;
4495
4496         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4497                 return -EINVAL;
4498
4499         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4500                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4501                 return -EINVAL;
4502         }
4503
4504         flags |= HIST_FIELD_FL_VAR;
4505         hist_data->n_vars++;
4506         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4507                 return -EINVAL;
4508
4509         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4510 }
4511
4512 static int create_val_fields(struct hist_trigger_data *hist_data,
4513                              struct trace_event_file *file)
4514 {
4515         char *fields_str, *field_str;
4516         unsigned int i, j = 1;
4517         int ret;
4518
4519         ret = create_hitcount_val(hist_data);
4520         if (ret)
4521                 goto out;
4522
4523         fields_str = hist_data->attrs->vals_str;
4524         if (!fields_str)
4525                 goto out;
4526
4527         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4528                      j < TRACING_MAP_VALS_MAX; i++) {
4529                 field_str = strsep(&fields_str, ",");
4530                 if (!field_str)
4531                         break;
4532
4533                 if (strcmp(field_str, "hitcount") == 0)
4534                         continue;
4535
4536                 ret = create_val_field(hist_data, j++, file, field_str);
4537                 if (ret)
4538                         goto out;
4539         }
4540
4541         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4542                 ret = -EINVAL;
4543  out:
4544         return ret;
4545 }
4546
4547 static int create_key_field(struct hist_trigger_data *hist_data,
4548                             unsigned int key_idx,
4549                             unsigned int key_offset,
4550                             struct trace_event_file *file,
4551                             char *field_str)
4552 {
4553         struct trace_array *tr = hist_data->event_file->tr;
4554         struct hist_field *hist_field = NULL;
4555         unsigned long flags = 0;
4556         unsigned int key_size;
4557         int ret = 0;
4558
4559         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4560                 return -EINVAL;
4561
4562         flags |= HIST_FIELD_FL_KEY;
4563
4564         if (strcmp(field_str, "stacktrace") == 0) {
4565                 flags |= HIST_FIELD_FL_STACKTRACE;
4566                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4567                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4568         } else {
4569                 hist_field = parse_expr(hist_data, file, field_str, flags,
4570                                         NULL, 0);
4571                 if (IS_ERR(hist_field)) {
4572                         ret = PTR_ERR(hist_field);
4573                         goto out;
4574                 }
4575
4576                 if (field_has_hist_vars(hist_field, 0)) {
4577                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4578                         destroy_hist_field(hist_field, 0);
4579                         ret = -EINVAL;
4580                         goto out;
4581                 }
4582
4583                 key_size = hist_field->size;
4584         }
4585
4586         hist_data->fields[key_idx] = hist_field;
4587
4588         key_size = ALIGN(key_size, sizeof(u64));
4589         hist_data->fields[key_idx]->size = key_size;
4590         hist_data->fields[key_idx]->offset = key_offset;
4591
4592         hist_data->key_size += key_size;
4593
4594         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4595                 ret = -EINVAL;
4596                 goto out;
4597         }
4598
4599         hist_data->n_keys++;
4600         hist_data->n_fields++;
4601
4602         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4603                 return -EINVAL;
4604
4605         ret = key_size;
4606  out:
4607         return ret;
4608 }
4609
4610 static int create_key_fields(struct hist_trigger_data *hist_data,
4611                              struct trace_event_file *file)
4612 {
4613         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4614         char *fields_str, *field_str;
4615         int ret = -EINVAL;
4616
4617         fields_str = hist_data->attrs->keys_str;
4618         if (!fields_str)
4619                 goto out;
4620
4621         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4622                 field_str = strsep(&fields_str, ",");
4623                 if (!field_str)
4624                         break;
4625                 ret = create_key_field(hist_data, i, key_offset,
4626                                        file, field_str);
4627                 if (ret < 0)
4628                         goto out;
4629                 key_offset += ret;
4630         }
4631         if (fields_str) {
4632                 ret = -EINVAL;
4633                 goto out;
4634         }
4635         ret = 0;
4636  out:
4637         return ret;
4638 }
4639
4640 static int create_var_fields(struct hist_trigger_data *hist_data,
4641                              struct trace_event_file *file)
4642 {
4643         unsigned int i, j = hist_data->n_vals;
4644         int ret = 0;
4645
4646         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4647
4648         for (i = 0; i < n_vars; i++) {
4649                 char *var_name = hist_data->attrs->var_defs.name[i];
4650                 char *expr = hist_data->attrs->var_defs.expr[i];
4651
4652                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4653                 if (ret)
4654                         goto out;
4655         }
4656  out:
4657         return ret;
4658 }
4659
4660 static void free_var_defs(struct hist_trigger_data *hist_data)
4661 {
4662         unsigned int i;
4663
4664         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4665                 kfree(hist_data->attrs->var_defs.name[i]);
4666                 kfree(hist_data->attrs->var_defs.expr[i]);
4667         }
4668
4669         hist_data->attrs->var_defs.n_vars = 0;
4670 }
4671
4672 static int parse_var_defs(struct hist_trigger_data *hist_data)
4673 {
4674         struct trace_array *tr = hist_data->event_file->tr;
4675         char *s, *str, *var_name, *field_str;
4676         unsigned int i, j, n_vars = 0;
4677         int ret = 0;
4678
4679         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4680                 str = hist_data->attrs->assignment_str[i];
4681                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4682                         field_str = strsep(&str, ",");
4683                         if (!field_str)
4684                                 break;
4685
4686                         var_name = strsep(&field_str, "=");
4687                         if (!var_name || !field_str) {
4688                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4689                                          errpos(var_name));
4690                                 ret = -EINVAL;
4691                                 goto free;
4692                         }
4693
4694                         if (n_vars == TRACING_MAP_VARS_MAX) {
4695                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4696                                 ret = -EINVAL;
4697                                 goto free;
4698                         }
4699
4700                         s = kstrdup(var_name, GFP_KERNEL);
4701                         if (!s) {
4702                                 ret = -ENOMEM;
4703                                 goto free;
4704                         }
4705                         hist_data->attrs->var_defs.name[n_vars] = s;
4706
4707                         s = kstrdup(field_str, GFP_KERNEL);
4708                         if (!s) {
4709                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4710                                 ret = -ENOMEM;
4711                                 goto free;
4712                         }
4713                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4714
4715                         hist_data->attrs->var_defs.n_vars = n_vars;
4716                 }
4717         }
4718
4719         return ret;
4720  free:
4721         free_var_defs(hist_data);
4722
4723         return ret;
4724 }
4725
4726 static int create_hist_fields(struct hist_trigger_data *hist_data,
4727                               struct trace_event_file *file)
4728 {
4729         int ret;
4730
4731         ret = parse_var_defs(hist_data);
4732         if (ret)
4733                 goto out;
4734
4735         ret = create_val_fields(hist_data, file);
4736         if (ret)
4737                 goto out;
4738
4739         ret = create_var_fields(hist_data, file);
4740         if (ret)
4741                 goto out;
4742
4743         ret = create_key_fields(hist_data, file);
4744         if (ret)
4745                 goto out;
4746  out:
4747         free_var_defs(hist_data);
4748
4749         return ret;
4750 }
4751
4752 static int is_descending(struct trace_array *tr, const char *str)
4753 {
4754         if (!str)
4755                 return 0;
4756
4757         if (strcmp(str, "descending") == 0)
4758                 return 1;
4759
4760         if (strcmp(str, "ascending") == 0)
4761                 return 0;
4762
4763         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4764
4765         return -EINVAL;
4766 }
4767
4768 static int create_sort_keys(struct hist_trigger_data *hist_data)
4769 {
4770         struct trace_array *tr = hist_data->event_file->tr;
4771         char *fields_str = hist_data->attrs->sort_key_str;
4772         struct tracing_map_sort_key *sort_key;
4773         int descending, ret = 0;
4774         unsigned int i, j, k;
4775
4776         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4777
4778         if (!fields_str)
4779                 goto out;
4780
4781         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4782                 struct hist_field *hist_field;
4783                 char *field_str, *field_name;
4784                 const char *test_name;
4785
4786                 sort_key = &hist_data->sort_keys[i];
4787
4788                 field_str = strsep(&fields_str, ",");
4789                 if (!field_str)
4790                         break;
4791
4792                 if (!*field_str) {
4793                         ret = -EINVAL;
4794                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4795                         break;
4796                 }
4797
4798                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4799                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4800                         ret = -EINVAL;
4801                         break;
4802                 }
4803
4804                 field_name = strsep(&field_str, ".");
4805                 if (!field_name || !*field_name) {
4806                         ret = -EINVAL;
4807                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4808                         break;
4809                 }
4810
4811                 if (strcmp(field_name, "hitcount") == 0) {
4812                         descending = is_descending(tr, field_str);
4813                         if (descending < 0) {
4814                                 ret = descending;
4815                                 break;
4816                         }
4817                         sort_key->descending = descending;
4818                         continue;
4819                 }
4820
4821                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4822                         unsigned int idx;
4823
4824                         hist_field = hist_data->fields[j];
4825                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4826                                 continue;
4827
4828                         idx = k++;
4829
4830                         test_name = hist_field_name(hist_field, 0);
4831
4832                         if (strcmp(field_name, test_name) == 0) {
4833                                 sort_key->field_idx = idx;
4834                                 descending = is_descending(tr, field_str);
4835                                 if (descending < 0) {
4836                                         ret = descending;
4837                                         goto out;
4838                                 }
4839                                 sort_key->descending = descending;
4840                                 break;
4841                         }
4842                 }
4843                 if (j == hist_data->n_fields) {
4844                         ret = -EINVAL;
4845                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4846                         break;
4847                 }
4848         }
4849
4850         hist_data->n_sort_keys = i;
4851  out:
4852         return ret;
4853 }
4854
4855 static void destroy_actions(struct hist_trigger_data *hist_data)
4856 {
4857         unsigned int i;
4858
4859         for (i = 0; i < hist_data->n_actions; i++) {
4860                 struct action_data *data = hist_data->actions[i];
4861
4862                 if (data->handler == HANDLER_ONMATCH)
4863                         onmatch_destroy(data);
4864                 else if (data->handler == HANDLER_ONMAX ||
4865                          data->handler == HANDLER_ONCHANGE)
4866                         track_data_destroy(hist_data, data);
4867                 else
4868                         kfree(data);
4869         }
4870 }
4871
4872 static int parse_actions(struct hist_trigger_data *hist_data)
4873 {
4874         struct trace_array *tr = hist_data->event_file->tr;
4875         struct action_data *data;
4876         unsigned int i;
4877         int ret = 0;
4878         char *str;
4879         int len;
4880
4881         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4882                 str = hist_data->attrs->action_str[i];
4883
4884                 if ((len = str_has_prefix(str, "onmatch("))) {
4885                         char *action_str = str + len;
4886
4887                         data = onmatch_parse(tr, action_str);
4888                         if (IS_ERR(data)) {
4889                                 ret = PTR_ERR(data);
4890                                 break;
4891                         }
4892                 } else if ((len = str_has_prefix(str, "onmax("))) {
4893                         char *action_str = str + len;
4894
4895                         data = track_data_parse(hist_data, action_str,
4896                                                 HANDLER_ONMAX);
4897                         if (IS_ERR(data)) {
4898                                 ret = PTR_ERR(data);
4899                                 break;
4900                         }
4901                 } else if ((len = str_has_prefix(str, "onchange("))) {
4902                         char *action_str = str + len;
4903
4904                         data = track_data_parse(hist_data, action_str,
4905                                                 HANDLER_ONCHANGE);
4906                         if (IS_ERR(data)) {
4907                                 ret = PTR_ERR(data);
4908                                 break;
4909                         }
4910                 } else {
4911                         ret = -EINVAL;
4912                         break;
4913                 }
4914
4915                 hist_data->actions[hist_data->n_actions++] = data;
4916         }
4917
4918         return ret;
4919 }
4920
4921 static int create_actions(struct hist_trigger_data *hist_data)
4922 {
4923         struct action_data *data;
4924         unsigned int i;
4925         int ret = 0;
4926
4927         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4928                 data = hist_data->actions[i];
4929
4930                 if (data->handler == HANDLER_ONMATCH) {
4931                         ret = onmatch_create(hist_data, data);
4932                         if (ret)
4933                                 break;
4934                 } else if (data->handler == HANDLER_ONMAX ||
4935                            data->handler == HANDLER_ONCHANGE) {
4936                         ret = track_data_create(hist_data, data);
4937                         if (ret)
4938                                 break;
4939                 } else {
4940                         ret = -EINVAL;
4941                         break;
4942                 }
4943         }
4944
4945         return ret;
4946 }
4947
4948 static void print_actions(struct seq_file *m,
4949                           struct hist_trigger_data *hist_data,
4950                           struct tracing_map_elt *elt)
4951 {
4952         unsigned int i;
4953
4954         for (i = 0; i < hist_data->n_actions; i++) {
4955                 struct action_data *data = hist_data->actions[i];
4956
4957                 if (data->action == ACTION_SNAPSHOT)
4958                         continue;
4959
4960                 if (data->handler == HANDLER_ONMAX ||
4961                     data->handler == HANDLER_ONCHANGE)
4962                         track_data_print(m, hist_data, elt, data);
4963         }
4964 }
4965
4966 static void print_action_spec(struct seq_file *m,
4967                               struct hist_trigger_data *hist_data,
4968                               struct action_data *data)
4969 {
4970         unsigned int i;
4971
4972         if (data->action == ACTION_SAVE) {
4973                 for (i = 0; i < hist_data->n_save_vars; i++) {
4974                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4975                         if (i < hist_data->n_save_vars - 1)
4976                                 seq_puts(m, ",");
4977                 }
4978         } else if (data->action == ACTION_TRACE) {
4979                 if (data->use_trace_keyword)
4980                         seq_printf(m, "%s", data->synth_event_name);
4981                 for (i = 0; i < data->n_params; i++) {
4982                         if (i || data->use_trace_keyword)
4983                                 seq_puts(m, ",");
4984                         seq_printf(m, "%s", data->params[i]);
4985                 }
4986         }
4987 }
4988
4989 static void print_track_data_spec(struct seq_file *m,
4990                                   struct hist_trigger_data *hist_data,
4991                                   struct action_data *data)
4992 {
4993         if (data->handler == HANDLER_ONMAX)
4994                 seq_puts(m, ":onmax(");
4995         else if (data->handler == HANDLER_ONCHANGE)
4996                 seq_puts(m, ":onchange(");
4997         seq_printf(m, "%s", data->track_data.var_str);
4998         seq_printf(m, ").%s(", data->action_name);
4999
5000         print_action_spec(m, hist_data, data);
5001
5002         seq_puts(m, ")");
5003 }
5004
5005 static void print_onmatch_spec(struct seq_file *m,
5006                                struct hist_trigger_data *hist_data,
5007                                struct action_data *data)
5008 {
5009         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
5010                    data->match_data.event);
5011
5012         seq_printf(m, "%s(", data->action_name);
5013
5014         print_action_spec(m, hist_data, data);
5015
5016         seq_puts(m, ")");
5017 }
5018
5019 static bool actions_match(struct hist_trigger_data *hist_data,
5020                           struct hist_trigger_data *hist_data_test)
5021 {
5022         unsigned int i, j;
5023
5024         if (hist_data->n_actions != hist_data_test->n_actions)
5025                 return false;
5026
5027         for (i = 0; i < hist_data->n_actions; i++) {
5028                 struct action_data *data = hist_data->actions[i];
5029                 struct action_data *data_test = hist_data_test->actions[i];
5030                 char *action_name, *action_name_test;
5031
5032                 if (data->handler != data_test->handler)
5033                         return false;
5034                 if (data->action != data_test->action)
5035                         return false;
5036
5037                 if (data->n_params != data_test->n_params)
5038                         return false;
5039
5040                 for (j = 0; j < data->n_params; j++) {
5041                         if (strcmp(data->params[j], data_test->params[j]) != 0)
5042                                 return false;
5043                 }
5044
5045                 if (data->use_trace_keyword)
5046                         action_name = data->synth_event_name;
5047                 else
5048                         action_name = data->action_name;
5049
5050                 if (data_test->use_trace_keyword)
5051                         action_name_test = data_test->synth_event_name;
5052                 else
5053                         action_name_test = data_test->action_name;
5054
5055                 if (strcmp(action_name, action_name_test) != 0)
5056                         return false;
5057
5058                 if (data->handler == HANDLER_ONMATCH) {
5059                         if (strcmp(data->match_data.event_system,
5060                                    data_test->match_data.event_system) != 0)
5061                                 return false;
5062                         if (strcmp(data->match_data.event,
5063                                    data_test->match_data.event) != 0)
5064                                 return false;
5065                 } else if (data->handler == HANDLER_ONMAX ||
5066                            data->handler == HANDLER_ONCHANGE) {
5067                         if (strcmp(data->track_data.var_str,
5068                                    data_test->track_data.var_str) != 0)
5069                                 return false;
5070                 }
5071         }
5072
5073         return true;
5074 }
5075
5076
5077 static void print_actions_spec(struct seq_file *m,
5078                                struct hist_trigger_data *hist_data)
5079 {
5080         unsigned int i;
5081
5082         for (i = 0; i < hist_data->n_actions; i++) {
5083                 struct action_data *data = hist_data->actions[i];
5084
5085                 if (data->handler == HANDLER_ONMATCH)
5086                         print_onmatch_spec(m, hist_data, data);
5087                 else if (data->handler == HANDLER_ONMAX ||
5088                          data->handler == HANDLER_ONCHANGE)
5089                         print_track_data_spec(m, hist_data, data);
5090         }
5091 }
5092
5093 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5094 {
5095         unsigned int i;
5096
5097         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5098                 kfree(hist_data->field_var_hists[i]->cmd);
5099                 kfree(hist_data->field_var_hists[i]);
5100         }
5101 }
5102
5103 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5104 {
5105         if (!hist_data)
5106                 return;
5107
5108         destroy_hist_trigger_attrs(hist_data->attrs);
5109         destroy_hist_fields(hist_data);
5110         tracing_map_destroy(hist_data->map);
5111
5112         destroy_actions(hist_data);
5113         destroy_field_vars(hist_data);
5114         destroy_field_var_hists(hist_data);
5115
5116         kfree(hist_data);
5117 }
5118
5119 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5120 {
5121         struct tracing_map *map = hist_data->map;
5122         struct ftrace_event_field *field;
5123         struct hist_field *hist_field;
5124         int i, idx = 0;
5125
5126         for_each_hist_field(i, hist_data) {
5127                 hist_field = hist_data->fields[i];
5128                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5129                         tracing_map_cmp_fn_t cmp_fn;
5130
5131                         field = hist_field->field;
5132
5133                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5134                                 cmp_fn = tracing_map_cmp_none;
5135                         else if (!field)
5136                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5137                                                              hist_field->is_signed);
5138                         else if (is_string_field(field))
5139                                 cmp_fn = tracing_map_cmp_string;
5140                         else
5141                                 cmp_fn = tracing_map_cmp_num(field->size,
5142                                                              field->is_signed);
5143                         idx = tracing_map_add_key_field(map,
5144                                                         hist_field->offset,
5145                                                         cmp_fn);
5146                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5147                         idx = tracing_map_add_sum_field(map);
5148
5149                 if (idx < 0)
5150                         return idx;
5151
5152                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5153                         idx = tracing_map_add_var(map);
5154                         if (idx < 0)
5155                                 return idx;
5156                         hist_field->var.idx = idx;
5157                         hist_field->var.hist_data = hist_data;
5158                 }
5159         }
5160
5161         return 0;
5162 }
5163
5164 static struct hist_trigger_data *
5165 create_hist_data(unsigned int map_bits,
5166                  struct hist_trigger_attrs *attrs,
5167                  struct trace_event_file *file,
5168                  bool remove)
5169 {
5170         const struct tracing_map_ops *map_ops = NULL;
5171         struct hist_trigger_data *hist_data;
5172         int ret = 0;
5173
5174         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5175         if (!hist_data)
5176                 return ERR_PTR(-ENOMEM);
5177
5178         hist_data->attrs = attrs;
5179         hist_data->remove = remove;
5180         hist_data->event_file = file;
5181
5182         ret = parse_actions(hist_data);
5183         if (ret)
5184                 goto free;
5185
5186         ret = create_hist_fields(hist_data, file);
5187         if (ret)
5188                 goto free;
5189
5190         ret = create_sort_keys(hist_data);
5191         if (ret)
5192                 goto free;
5193
5194         map_ops = &hist_trigger_elt_data_ops;
5195
5196         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5197                                             map_ops, hist_data);
5198         if (IS_ERR(hist_data->map)) {
5199                 ret = PTR_ERR(hist_data->map);
5200                 hist_data->map = NULL;
5201                 goto free;
5202         }
5203
5204         ret = create_tracing_map_fields(hist_data);
5205         if (ret)
5206                 goto free;
5207  out:
5208         return hist_data;
5209  free:
5210         hist_data->attrs = NULL;
5211
5212         destroy_hist_data(hist_data);
5213
5214         hist_data = ERR_PTR(ret);
5215
5216         goto out;
5217 }
5218
5219 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5220                                     struct tracing_map_elt *elt, void *rec,
5221                                     struct ring_buffer_event *rbe,
5222                                     u64 *var_ref_vals)
5223 {
5224         struct hist_elt_data *elt_data;
5225         struct hist_field *hist_field;
5226         unsigned int i, var_idx;
5227         u64 hist_val;
5228
5229         elt_data = elt->private_data;
5230         elt_data->var_ref_vals = var_ref_vals;
5231
5232         for_each_hist_val_field(i, hist_data) {
5233                 hist_field = hist_data->fields[i];
5234                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5235                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5236                         var_idx = hist_field->var.idx;
5237                         tracing_map_set_var(elt, var_idx, hist_val);
5238                         continue;
5239                 }
5240                 tracing_map_update_sum(elt, i, hist_val);
5241         }
5242
5243         for_each_hist_key_field(i, hist_data) {
5244                 hist_field = hist_data->fields[i];
5245                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5246                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5247                         var_idx = hist_field->var.idx;
5248                         tracing_map_set_var(elt, var_idx, hist_val);
5249                 }
5250         }
5251
5252         update_field_vars(hist_data, elt, rbe, rec);
5253 }
5254
5255 static inline void add_to_key(char *compound_key, void *key,
5256                               struct hist_field *key_field, void *rec)
5257 {
5258         size_t size = key_field->size;
5259
5260         if (key_field->flags & HIST_FIELD_FL_STRING) {
5261                 struct ftrace_event_field *field;
5262
5263                 field = key_field->field;
5264                 if (field->filter_type == FILTER_DYN_STRING)
5265                         size = *(u32 *)(rec + field->offset) >> 16;
5266                 else if (field->filter_type == FILTER_PTR_STRING)
5267                         size = strlen(key);
5268                 else if (field->filter_type == FILTER_STATIC_STRING)
5269                         size = field->size;
5270
5271                 /* ensure NULL-termination */
5272                 if (size > key_field->size - 1)
5273                         size = key_field->size - 1;
5274
5275                 strncpy(compound_key + key_field->offset, (char *)key, size);
5276         } else
5277                 memcpy(compound_key + key_field->offset, key, size);
5278 }
5279
5280 static void
5281 hist_trigger_actions(struct hist_trigger_data *hist_data,
5282                      struct tracing_map_elt *elt, void *rec,
5283                      struct ring_buffer_event *rbe, void *key,
5284                      u64 *var_ref_vals)
5285 {
5286         struct action_data *data;
5287         unsigned int i;
5288
5289         for (i = 0; i < hist_data->n_actions; i++) {
5290                 data = hist_data->actions[i];
5291                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
5292         }
5293 }
5294
5295 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
5296                                struct ring_buffer_event *rbe)
5297 {
5298         struct hist_trigger_data *hist_data = data->private_data;
5299         bool use_compound_key = (hist_data->n_keys > 1);
5300         unsigned long entries[HIST_STACKTRACE_DEPTH];
5301         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5302         char compound_key[HIST_KEY_SIZE_MAX];
5303         struct tracing_map_elt *elt = NULL;
5304         struct hist_field *key_field;
5305         u64 field_contents;
5306         void *key = NULL;
5307         unsigned int i;
5308
5309         memset(compound_key, 0, hist_data->key_size);
5310
5311         for_each_hist_key_field(i, hist_data) {
5312                 key_field = hist_data->fields[i];
5313
5314                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5315                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5316                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5317                                          HIST_STACKTRACE_SKIP);
5318                         key = entries;
5319                 } else {
5320                         field_contents = key_field->fn(key_field, elt, rbe, rec);
5321                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5322                                 key = (void *)(unsigned long)field_contents;
5323                                 use_compound_key = true;
5324                         } else
5325                                 key = (void *)&field_contents;
5326                 }
5327
5328                 if (use_compound_key)
5329                         add_to_key(compound_key, key, key_field, rec);
5330         }
5331
5332         if (use_compound_key)
5333                 key = compound_key;
5334
5335         if (hist_data->n_var_refs &&
5336             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5337                 return;
5338
5339         elt = tracing_map_insert(hist_data->map, key);
5340         if (!elt)
5341                 return;
5342
5343         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
5344
5345         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5346                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
5347 }
5348
5349 static void hist_trigger_stacktrace_print(struct seq_file *m,
5350                                           unsigned long *stacktrace_entries,
5351                                           unsigned int max_entries)
5352 {
5353         char str[KSYM_SYMBOL_LEN];
5354         unsigned int spaces = 8;
5355         unsigned int i;
5356
5357         for (i = 0; i < max_entries; i++) {
5358                 if (!stacktrace_entries[i])
5359                         return;
5360
5361                 seq_printf(m, "%*c", 1 + spaces, ' ');
5362                 sprint_symbol(str, stacktrace_entries[i]);
5363                 seq_printf(m, "%s\n", str);
5364         }
5365 }
5366
5367 static void hist_trigger_print_key(struct seq_file *m,
5368                                    struct hist_trigger_data *hist_data,
5369                                    void *key,
5370                                    struct tracing_map_elt *elt)
5371 {
5372         struct hist_field *key_field;
5373         char str[KSYM_SYMBOL_LEN];
5374         bool multiline = false;
5375         const char *field_name;
5376         unsigned int i;
5377         u64 uval;
5378
5379         seq_puts(m, "{ ");
5380
5381         for_each_hist_key_field(i, hist_data) {
5382                 key_field = hist_data->fields[i];
5383
5384                 if (i > hist_data->n_vals)
5385                         seq_puts(m, ", ");
5386
5387                 field_name = hist_field_name(key_field, 0);
5388
5389                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5390                         uval = *(u64 *)(key + key_field->offset);
5391                         seq_printf(m, "%s: %llx", field_name, uval);
5392                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5393                         uval = *(u64 *)(key + key_field->offset);
5394                         sprint_symbol_no_offset(str, uval);
5395                         seq_printf(m, "%s: [%llx] %-45s", field_name,
5396                                    uval, str);
5397                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5398                         uval = *(u64 *)(key + key_field->offset);
5399                         sprint_symbol(str, uval);
5400                         seq_printf(m, "%s: [%llx] %-55s", field_name,
5401                                    uval, str);
5402                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5403                         struct hist_elt_data *elt_data = elt->private_data;
5404                         char *comm;
5405
5406                         if (WARN_ON_ONCE(!elt_data))
5407                                 return;
5408
5409                         comm = elt_data->comm;
5410
5411                         uval = *(u64 *)(key + key_field->offset);
5412                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5413                                    comm, uval);
5414                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5415                         const char *syscall_name;
5416
5417                         uval = *(u64 *)(key + key_field->offset);
5418                         syscall_name = get_syscall_name(uval);
5419                         if (!syscall_name)
5420                                 syscall_name = "unknown_syscall";
5421
5422                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5423                                    syscall_name, uval);
5424                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5425                         seq_puts(m, "stacktrace:\n");
5426                         hist_trigger_stacktrace_print(m,
5427                                                       key + key_field->offset,
5428                                                       HIST_STACKTRACE_DEPTH);
5429                         multiline = true;
5430                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5431                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5432                                    *(u64 *)(key + key_field->offset));
5433                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5434                         seq_printf(m, "%s: %-50s", field_name,
5435                                    (char *)(key + key_field->offset));
5436                 } else {
5437                         uval = *(u64 *)(key + key_field->offset);
5438                         seq_printf(m, "%s: %10llu", field_name, uval);
5439                 }
5440         }
5441
5442         if (!multiline)
5443                 seq_puts(m, " ");
5444
5445         seq_puts(m, "}");
5446 }
5447
5448 static void hist_trigger_entry_print(struct seq_file *m,
5449                                      struct hist_trigger_data *hist_data,
5450                                      void *key,
5451                                      struct tracing_map_elt *elt)
5452 {
5453         const char *field_name;
5454         unsigned int i;
5455
5456         hist_trigger_print_key(m, hist_data, key, elt);
5457
5458         seq_printf(m, " hitcount: %10llu",
5459                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5460
5461         for (i = 1; i < hist_data->n_vals; i++) {
5462                 field_name = hist_field_name(hist_data->fields[i], 0);
5463
5464                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5465                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5466                         continue;
5467
5468                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5469                         seq_printf(m, "  %s: %10llx", field_name,
5470                                    tracing_map_read_sum(elt, i));
5471                 } else {
5472                         seq_printf(m, "  %s: %10llu", field_name,
5473                                    tracing_map_read_sum(elt, i));
5474                 }
5475         }
5476
5477         print_actions(m, hist_data, elt);
5478
5479         seq_puts(m, "\n");
5480 }
5481
5482 static int print_entries(struct seq_file *m,
5483                          struct hist_trigger_data *hist_data)
5484 {
5485         struct tracing_map_sort_entry **sort_entries = NULL;
5486         struct tracing_map *map = hist_data->map;
5487         int i, n_entries;
5488
5489         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5490                                              hist_data->n_sort_keys,
5491                                              &sort_entries);
5492         if (n_entries < 0)
5493                 return n_entries;
5494
5495         for (i = 0; i < n_entries; i++)
5496                 hist_trigger_entry_print(m, hist_data,
5497                                          sort_entries[i]->key,
5498                                          sort_entries[i]->elt);
5499
5500         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5501
5502         return n_entries;
5503 }
5504
5505 static void hist_trigger_show(struct seq_file *m,
5506                               struct event_trigger_data *data, int n)
5507 {
5508         struct hist_trigger_data *hist_data;
5509         int n_entries;
5510
5511         if (n > 0)
5512                 seq_puts(m, "\n\n");
5513
5514         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5515         data->ops->print(m, data->ops, data);
5516         seq_puts(m, "#\n\n");
5517
5518         hist_data = data->private_data;
5519         n_entries = print_entries(m, hist_data);
5520         if (n_entries < 0)
5521                 n_entries = 0;
5522
5523         track_data_snapshot_print(m, hist_data);
5524
5525         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5526                    (u64)atomic64_read(&hist_data->map->hits),
5527                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5528 }
5529
5530 static int hist_show(struct seq_file *m, void *v)
5531 {
5532         struct event_trigger_data *data;
5533         struct trace_event_file *event_file;
5534         int n = 0, ret = 0;
5535
5536         mutex_lock(&event_mutex);
5537
5538         event_file = event_file_data(m->private);
5539         if (unlikely(!event_file)) {
5540                 ret = -ENODEV;
5541                 goto out_unlock;
5542         }
5543
5544         list_for_each_entry(data, &event_file->triggers, list) {
5545                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5546                         hist_trigger_show(m, data, n++);
5547         }
5548
5549  out_unlock:
5550         mutex_unlock(&event_mutex);
5551
5552         return ret;
5553 }
5554
5555 static int event_hist_open(struct inode *inode, struct file *file)
5556 {
5557         int ret;
5558
5559         ret = security_locked_down(LOCKDOWN_TRACEFS);
5560         if (ret)
5561                 return ret;
5562
5563         return single_open(file, hist_show, file);
5564 }
5565
5566 const struct file_operations event_hist_fops = {
5567         .open = event_hist_open,
5568         .read = seq_read,
5569         .llseek = seq_lseek,
5570         .release = single_release,
5571 };
5572
5573 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5574 {
5575         const char *field_name = hist_field_name(hist_field, 0);
5576
5577         if (hist_field->var.name)
5578                 seq_printf(m, "%s=", hist_field->var.name);
5579
5580         if (hist_field->flags & HIST_FIELD_FL_CPU)
5581                 seq_puts(m, "cpu");
5582         else if (field_name) {
5583                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5584                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5585                         seq_putc(m, '$');
5586                 seq_printf(m, "%s", field_name);
5587         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5588                 seq_puts(m, "common_timestamp");
5589
5590         if (hist_field->flags) {
5591                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5592                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5593                         const char *flags = get_hist_field_flags(hist_field);
5594
5595                         if (flags)
5596                                 seq_printf(m, ".%s", flags);
5597                 }
5598         }
5599 }
5600
5601 static int event_hist_trigger_print(struct seq_file *m,
5602                                     struct event_trigger_ops *ops,
5603                                     struct event_trigger_data *data)
5604 {
5605         struct hist_trigger_data *hist_data = data->private_data;
5606         struct hist_field *field;
5607         bool have_var = false;
5608         unsigned int i;
5609
5610         seq_puts(m, "hist:");
5611
5612         if (data->name)
5613                 seq_printf(m, "%s:", data->name);
5614
5615         seq_puts(m, "keys=");
5616
5617         for_each_hist_key_field(i, hist_data) {
5618                 field = hist_data->fields[i];
5619
5620                 if (i > hist_data->n_vals)
5621                         seq_puts(m, ",");
5622
5623                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5624                         seq_puts(m, "stacktrace");
5625                 else
5626                         hist_field_print(m, field);
5627         }
5628
5629         seq_puts(m, ":vals=");
5630
5631         for_each_hist_val_field(i, hist_data) {
5632                 field = hist_data->fields[i];
5633                 if (field->flags & HIST_FIELD_FL_VAR) {
5634                         have_var = true;
5635                         continue;
5636                 }
5637
5638                 if (i == HITCOUNT_IDX)
5639                         seq_puts(m, "hitcount");
5640                 else {
5641                         seq_puts(m, ",");
5642                         hist_field_print(m, field);
5643                 }
5644         }
5645
5646         if (have_var) {
5647                 unsigned int n = 0;
5648
5649                 seq_puts(m, ":");
5650
5651                 for_each_hist_val_field(i, hist_data) {
5652                         field = hist_data->fields[i];
5653
5654                         if (field->flags & HIST_FIELD_FL_VAR) {
5655                                 if (n++)
5656                                         seq_puts(m, ",");
5657                                 hist_field_print(m, field);
5658                         }
5659                 }
5660         }
5661
5662         seq_puts(m, ":sort=");
5663
5664         for (i = 0; i < hist_data->n_sort_keys; i++) {
5665                 struct tracing_map_sort_key *sort_key;
5666                 unsigned int idx, first_key_idx;
5667
5668                 /* skip VAR vals */
5669                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5670
5671                 sort_key = &hist_data->sort_keys[i];
5672                 idx = sort_key->field_idx;
5673
5674                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5675                         return -EINVAL;
5676
5677                 if (i > 0)
5678                         seq_puts(m, ",");
5679
5680                 if (idx == HITCOUNT_IDX)
5681                         seq_puts(m, "hitcount");
5682                 else {
5683                         if (idx >= first_key_idx)
5684                                 idx += hist_data->n_vars;
5685                         hist_field_print(m, hist_data->fields[idx]);
5686                 }
5687
5688                 if (sort_key->descending)
5689                         seq_puts(m, ".descending");
5690         }
5691         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5692         if (hist_data->enable_timestamps)
5693                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5694
5695         print_actions_spec(m, hist_data);
5696
5697         if (data->filter_str)
5698                 seq_printf(m, " if %s", data->filter_str);
5699
5700         if (data->paused)
5701                 seq_puts(m, " [paused]");
5702         else
5703                 seq_puts(m, " [active]");
5704
5705         seq_putc(m, '\n');
5706
5707         return 0;
5708 }
5709
5710 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5711                                    struct event_trigger_data *data)
5712 {
5713         struct hist_trigger_data *hist_data = data->private_data;
5714
5715         if (!data->ref && hist_data->attrs->name)
5716                 save_named_trigger(hist_data->attrs->name, data);
5717
5718         data->ref++;
5719
5720         return 0;
5721 }
5722
5723 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5724 {
5725         struct trace_event_file *file;
5726         unsigned int i;
5727         char *cmd;
5728         int ret;
5729
5730         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5731                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5732                 cmd = hist_data->field_var_hists[i]->cmd;
5733                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5734                                               "!hist", "hist", cmd);
5735         }
5736 }
5737
5738 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5739                                     struct event_trigger_data *data)
5740 {
5741         struct hist_trigger_data *hist_data = data->private_data;
5742
5743         if (WARN_ON_ONCE(data->ref <= 0))
5744                 return;
5745
5746         data->ref--;
5747         if (!data->ref) {
5748                 if (data->name)
5749                         del_named_trigger(data);
5750
5751                 trigger_data_free(data);
5752
5753                 remove_hist_vars(hist_data);
5754
5755                 unregister_field_var_hists(hist_data);
5756
5757                 destroy_hist_data(hist_data);
5758         }
5759 }
5760
5761 static struct event_trigger_ops event_hist_trigger_ops = {
5762         .func                   = event_hist_trigger,
5763         .print                  = event_hist_trigger_print,
5764         .init                   = event_hist_trigger_init,
5765         .free                   = event_hist_trigger_free,
5766 };
5767
5768 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5769                                          struct event_trigger_data *data)
5770 {
5771         data->ref++;
5772
5773         save_named_trigger(data->named_data->name, data);
5774
5775         event_hist_trigger_init(ops, data->named_data);
5776
5777         return 0;
5778 }
5779
5780 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5781                                           struct event_trigger_data *data)
5782 {
5783         if (WARN_ON_ONCE(data->ref <= 0))
5784                 return;
5785
5786         event_hist_trigger_free(ops, data->named_data);
5787
5788         data->ref--;
5789         if (!data->ref) {
5790                 del_named_trigger(data);
5791                 trigger_data_free(data);
5792         }
5793 }
5794
5795 static struct event_trigger_ops event_hist_trigger_named_ops = {
5796         .func                   = event_hist_trigger,
5797         .print                  = event_hist_trigger_print,
5798         .init                   = event_hist_trigger_named_init,
5799         .free                   = event_hist_trigger_named_free,
5800 };
5801
5802 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5803                                                             char *param)
5804 {
5805         return &event_hist_trigger_ops;
5806 }
5807
5808 static void hist_clear(struct event_trigger_data *data)
5809 {
5810         struct hist_trigger_data *hist_data = data->private_data;
5811
5812         if (data->name)
5813                 pause_named_trigger(data);
5814
5815         tracepoint_synchronize_unregister();
5816
5817         tracing_map_clear(hist_data->map);
5818
5819         if (data->name)
5820                 unpause_named_trigger(data);
5821 }
5822
5823 static bool compatible_field(struct ftrace_event_field *field,
5824                              struct ftrace_event_field *test_field)
5825 {
5826         if (field == test_field)
5827                 return true;
5828         if (field == NULL || test_field == NULL)
5829                 return false;
5830         if (strcmp(field->name, test_field->name) != 0)
5831                 return false;
5832         if (strcmp(field->type, test_field->type) != 0)
5833                 return false;
5834         if (field->size != test_field->size)
5835                 return false;
5836         if (field->is_signed != test_field->is_signed)
5837                 return false;
5838
5839         return true;
5840 }
5841
5842 static bool hist_trigger_match(struct event_trigger_data *data,
5843                                struct event_trigger_data *data_test,
5844                                struct event_trigger_data *named_data,
5845                                bool ignore_filter)
5846 {
5847         struct tracing_map_sort_key *sort_key, *sort_key_test;
5848         struct hist_trigger_data *hist_data, *hist_data_test;
5849         struct hist_field *key_field, *key_field_test;
5850         unsigned int i;
5851
5852         if (named_data && (named_data != data_test) &&
5853             (named_data != data_test->named_data))
5854                 return false;
5855
5856         if (!named_data && is_named_trigger(data_test))
5857                 return false;
5858
5859         hist_data = data->private_data;
5860         hist_data_test = data_test->private_data;
5861
5862         if (hist_data->n_vals != hist_data_test->n_vals ||
5863             hist_data->n_fields != hist_data_test->n_fields ||
5864             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5865                 return false;
5866
5867         if (!ignore_filter) {
5868                 if ((data->filter_str && !data_test->filter_str) ||
5869                    (!data->filter_str && data_test->filter_str))
5870                         return false;
5871         }
5872
5873         for_each_hist_field(i, hist_data) {
5874                 key_field = hist_data->fields[i];
5875                 key_field_test = hist_data_test->fields[i];
5876
5877                 if (key_field->flags != key_field_test->flags)
5878                         return false;
5879                 if (!compatible_field(key_field->field, key_field_test->field))
5880                         return false;
5881                 if (key_field->offset != key_field_test->offset)
5882                         return false;
5883                 if (key_field->size != key_field_test->size)
5884                         return false;
5885                 if (key_field->is_signed != key_field_test->is_signed)
5886                         return false;
5887                 if (!!key_field->var.name != !!key_field_test->var.name)
5888                         return false;
5889                 if (key_field->var.name &&
5890                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5891                         return false;
5892         }
5893
5894         for (i = 0; i < hist_data->n_sort_keys; i++) {
5895                 sort_key = &hist_data->sort_keys[i];
5896                 sort_key_test = &hist_data_test->sort_keys[i];
5897
5898                 if (sort_key->field_idx != sort_key_test->field_idx ||
5899                     sort_key->descending != sort_key_test->descending)
5900                         return false;
5901         }
5902
5903         if (!ignore_filter && data->filter_str &&
5904             (strcmp(data->filter_str, data_test->filter_str) != 0))
5905                 return false;
5906
5907         if (!actions_match(hist_data, hist_data_test))
5908                 return false;
5909
5910         return true;
5911 }
5912
5913 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5914                                  struct event_trigger_data *data,
5915                                  struct trace_event_file *file)
5916 {
5917         struct hist_trigger_data *hist_data = data->private_data;
5918         struct event_trigger_data *test, *named_data = NULL;
5919         struct trace_array *tr = file->tr;
5920         int ret = 0;
5921
5922         if (hist_data->attrs->name) {
5923                 named_data = find_named_trigger(hist_data->attrs->name);
5924                 if (named_data) {
5925                         if (!hist_trigger_match(data, named_data, named_data,
5926                                                 true)) {
5927                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5928                                 ret = -EINVAL;
5929                                 goto out;
5930                         }
5931                 }
5932         }
5933
5934         if (hist_data->attrs->name && !named_data)
5935                 goto new;
5936
5937         lockdep_assert_held(&event_mutex);
5938
5939         list_for_each_entry(test, &file->triggers, list) {
5940                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5941                         if (!hist_trigger_match(data, test, named_data, false))
5942                                 continue;
5943                         if (hist_data->attrs->pause)
5944                                 test->paused = true;
5945                         else if (hist_data->attrs->cont)
5946                                 test->paused = false;
5947                         else if (hist_data->attrs->clear)
5948                                 hist_clear(test);
5949                         else {
5950                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5951                                 ret = -EEXIST;
5952                         }
5953                         goto out;
5954                 }
5955         }
5956  new:
5957         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5958                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5959                 ret = -ENOENT;
5960                 goto out;
5961         }
5962
5963         if (hist_data->attrs->pause)
5964                 data->paused = true;
5965
5966         if (named_data) {
5967                 data->private_data = named_data->private_data;
5968                 set_named_trigger_data(data, named_data);
5969                 data->ops = &event_hist_trigger_named_ops;
5970         }
5971
5972         if (data->ops->init) {
5973                 ret = data->ops->init(data->ops, data);
5974                 if (ret < 0)
5975                         goto out;
5976         }
5977
5978         if (hist_data->enable_timestamps) {
5979                 char *clock = hist_data->attrs->clock;
5980
5981                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5982                 if (ret) {
5983                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5984                         goto out;
5985                 }
5986
5987                 tracing_set_time_stamp_abs(file->tr, true);
5988         }
5989
5990         if (named_data)
5991                 destroy_hist_data(hist_data);
5992
5993         ret++;
5994  out:
5995         return ret;
5996 }
5997
5998 static int hist_trigger_enable(struct event_trigger_data *data,
5999                                struct trace_event_file *file)
6000 {
6001         int ret = 0;
6002
6003         list_add_tail_rcu(&data->list, &file->triggers);
6004
6005         update_cond_flag(file);
6006
6007         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6008                 list_del_rcu(&data->list);
6009                 update_cond_flag(file);
6010                 ret--;
6011         }
6012
6013         return ret;
6014 }
6015
6016 static bool have_hist_trigger_match(struct event_trigger_data *data,
6017                                     struct trace_event_file *file)
6018 {
6019         struct hist_trigger_data *hist_data = data->private_data;
6020         struct event_trigger_data *test, *named_data = NULL;
6021         bool match = false;
6022
6023         lockdep_assert_held(&event_mutex);
6024
6025         if (hist_data->attrs->name)
6026                 named_data = find_named_trigger(hist_data->attrs->name);
6027
6028         list_for_each_entry(test, &file->triggers, list) {
6029                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6030                         if (hist_trigger_match(data, test, named_data, false)) {
6031                                 match = true;
6032                                 break;
6033                         }
6034                 }
6035         }
6036
6037         return match;
6038 }
6039
6040 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6041                                     struct trace_event_file *file)
6042 {
6043         struct hist_trigger_data *hist_data = data->private_data;
6044         struct event_trigger_data *test, *named_data = NULL;
6045
6046         lockdep_assert_held(&event_mutex);
6047
6048         if (hist_data->attrs->name)
6049                 named_data = find_named_trigger(hist_data->attrs->name);
6050
6051         list_for_each_entry(test, &file->triggers, list) {
6052                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6053                         if (!hist_trigger_match(data, test, named_data, false))
6054                                 continue;
6055                         hist_data = test->private_data;
6056                         if (check_var_refs(hist_data))
6057                                 return true;
6058                         break;
6059                 }
6060         }
6061
6062         return false;
6063 }
6064
6065 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
6066                                     struct event_trigger_data *data,
6067                                     struct trace_event_file *file)
6068 {
6069         struct hist_trigger_data *hist_data = data->private_data;
6070         struct event_trigger_data *test, *named_data = NULL;
6071         bool unregistered = false;
6072
6073         lockdep_assert_held(&event_mutex);
6074
6075         if (hist_data->attrs->name)
6076                 named_data = find_named_trigger(hist_data->attrs->name);
6077
6078         list_for_each_entry(test, &file->triggers, list) {
6079                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6080                         if (!hist_trigger_match(data, test, named_data, false))
6081                                 continue;
6082                         unregistered = true;
6083                         list_del_rcu(&test->list);
6084                         trace_event_trigger_enable_disable(file, 0);
6085                         update_cond_flag(file);
6086                         break;
6087                 }
6088         }
6089
6090         if (unregistered && test->ops->free)
6091                 test->ops->free(test->ops, test);
6092
6093         if (hist_data->enable_timestamps) {
6094                 if (!hist_data->remove || unregistered)
6095                         tracing_set_time_stamp_abs(file->tr, false);
6096         }
6097 }
6098
6099 static bool hist_file_check_refs(struct trace_event_file *file)
6100 {
6101         struct hist_trigger_data *hist_data;
6102         struct event_trigger_data *test;
6103
6104         lockdep_assert_held(&event_mutex);
6105
6106         list_for_each_entry(test, &file->triggers, list) {
6107                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6108                         hist_data = test->private_data;
6109                         if (check_var_refs(hist_data))
6110                                 return true;
6111                 }
6112         }
6113
6114         return false;
6115 }
6116
6117 static void hist_unreg_all(struct trace_event_file *file)
6118 {
6119         struct event_trigger_data *test, *n;
6120         struct hist_trigger_data *hist_data;
6121         struct synth_event *se;
6122         const char *se_name;
6123
6124         lockdep_assert_held(&event_mutex);
6125
6126         if (hist_file_check_refs(file))
6127                 return;
6128
6129         list_for_each_entry_safe(test, n, &file->triggers, list) {
6130                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6131                         hist_data = test->private_data;
6132                         list_del_rcu(&test->list);
6133                         trace_event_trigger_enable_disable(file, 0);
6134
6135                         se_name = trace_event_name(file->event_call);
6136                         se = find_synth_event(se_name);
6137                         if (se)
6138                                 se->ref--;
6139
6140                         update_cond_flag(file);
6141                         if (hist_data->enable_timestamps)
6142                                 tracing_set_time_stamp_abs(file->tr, false);
6143                         if (test->ops->free)
6144                                 test->ops->free(test->ops, test);
6145                 }
6146         }
6147 }
6148
6149 static int event_hist_trigger_func(struct event_command *cmd_ops,
6150                                    struct trace_event_file *file,
6151                                    char *glob, char *cmd, char *param)
6152 {
6153         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6154         struct event_trigger_data *trigger_data;
6155         struct hist_trigger_attrs *attrs;
6156         struct event_trigger_ops *trigger_ops;
6157         struct hist_trigger_data *hist_data;
6158         struct synth_event *se;
6159         const char *se_name;
6160         bool remove = false;
6161         char *trigger, *p;
6162         int ret = 0;
6163
6164         lockdep_assert_held(&event_mutex);
6165
6166         if (glob && strlen(glob)) {
6167                 hist_err_clear();
6168                 last_cmd_set(file, param);
6169         }
6170
6171         if (!param)
6172                 return -EINVAL;
6173
6174         if (glob[0] == '!')
6175                 remove = true;
6176
6177         /*
6178          * separate the trigger from the filter (k:v [if filter])
6179          * allowing for whitespace in the trigger
6180          */
6181         p = trigger = param;
6182         do {
6183                 p = strstr(p, "if");
6184                 if (!p)
6185                         break;
6186                 if (p == param)
6187                         return -EINVAL;
6188                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6189                         p++;
6190                         continue;
6191                 }
6192                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6193                         return -EINVAL;
6194                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6195                         p++;
6196                         continue;
6197                 }
6198                 break;
6199         } while (p);
6200
6201         if (!p)
6202                 param = NULL;
6203         else {
6204                 *(p - 1) = '\0';
6205                 param = strstrip(p);
6206                 trigger = strstrip(trigger);
6207         }
6208
6209         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6210         if (IS_ERR(attrs))
6211                 return PTR_ERR(attrs);
6212
6213         if (attrs->map_bits)
6214                 hist_trigger_bits = attrs->map_bits;
6215
6216         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6217         if (IS_ERR(hist_data)) {
6218                 destroy_hist_trigger_attrs(attrs);
6219                 return PTR_ERR(hist_data);
6220         }
6221
6222         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6223
6224         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6225         if (!trigger_data) {
6226                 ret = -ENOMEM;
6227                 goto out_free;
6228         }
6229
6230         trigger_data->count = -1;
6231         trigger_data->ops = trigger_ops;
6232         trigger_data->cmd_ops = cmd_ops;
6233
6234         INIT_LIST_HEAD(&trigger_data->list);
6235         RCU_INIT_POINTER(trigger_data->filter, NULL);
6236
6237         trigger_data->private_data = hist_data;
6238
6239         /* if param is non-empty, it's supposed to be a filter */
6240         if (param && cmd_ops->set_filter) {
6241                 ret = cmd_ops->set_filter(param, trigger_data, file);
6242                 if (ret < 0)
6243                         goto out_free;
6244         }
6245
6246         if (remove) {
6247                 if (!have_hist_trigger_match(trigger_data, file))
6248                         goto out_free;
6249
6250                 if (hist_trigger_check_refs(trigger_data, file)) {
6251                         ret = -EBUSY;
6252                         goto out_free;
6253                 }
6254
6255                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6256                 se_name = trace_event_name(file->event_call);
6257                 se = find_synth_event(se_name);
6258                 if (se)
6259                         se->ref--;
6260                 ret = 0;
6261                 goto out_free;
6262         }
6263
6264         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6265         /*
6266          * The above returns on success the # of triggers registered,
6267          * but if it didn't register any it returns zero.  Consider no
6268          * triggers registered a failure too.
6269          */
6270         if (!ret) {
6271                 if (!(attrs->pause || attrs->cont || attrs->clear))
6272                         ret = -ENOENT;
6273                 goto out_free;
6274         } else if (ret < 0)
6275                 goto out_free;
6276
6277         if (get_named_trigger_data(trigger_data))
6278                 goto enable;
6279
6280         if (has_hist_vars(hist_data))
6281                 save_hist_vars(hist_data);
6282
6283         ret = create_actions(hist_data);
6284         if (ret)
6285                 goto out_unreg;
6286
6287         ret = tracing_map_init(hist_data->map);
6288         if (ret)
6289                 goto out_unreg;
6290 enable:
6291         ret = hist_trigger_enable(trigger_data, file);
6292         if (ret)
6293                 goto out_unreg;
6294
6295         se_name = trace_event_name(file->event_call);
6296         se = find_synth_event(se_name);
6297         if (se)
6298                 se->ref++;
6299         /* Just return zero, not the number of registered triggers */
6300         ret = 0;
6301  out:
6302         if (ret == 0)
6303                 hist_err_clear();
6304
6305         return ret;
6306  out_unreg:
6307         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6308  out_free:
6309         if (cmd_ops->set_filter)
6310                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6311
6312         remove_hist_vars(hist_data);
6313
6314         kfree(trigger_data);
6315
6316         destroy_hist_data(hist_data);
6317         goto out;
6318 }
6319
6320 static struct event_command trigger_hist_cmd = {
6321         .name                   = "hist",
6322         .trigger_type           = ETT_EVENT_HIST,
6323         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6324         .func                   = event_hist_trigger_func,
6325         .reg                    = hist_register_trigger,
6326         .unreg                  = hist_unregister_trigger,
6327         .unreg_all              = hist_unreg_all,
6328         .get_trigger_ops        = event_hist_get_trigger_ops,
6329         .set_filter             = set_trigger_filter,
6330 };
6331
6332 __init int register_trigger_hist_cmd(void)
6333 {
6334         int ret;
6335
6336         ret = register_event_command(&trigger_hist_cmd);
6337         WARN_ON(ret < 0);
6338
6339         return ret;
6340 }
6341
6342 static void
6343 hist_enable_trigger(struct event_trigger_data *data, void *rec,
6344                     struct ring_buffer_event *event)
6345 {
6346         struct enable_trigger_data *enable_data = data->private_data;
6347         struct event_trigger_data *test;
6348
6349         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6350                                 lockdep_is_held(&event_mutex)) {
6351                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6352                         if (enable_data->enable)
6353                                 test->paused = false;
6354                         else
6355                                 test->paused = true;
6356                 }
6357         }
6358 }
6359
6360 static void
6361 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
6362                           struct ring_buffer_event *event)
6363 {
6364         if (!data->count)
6365                 return;
6366
6367         if (data->count != -1)
6368                 (data->count)--;
6369
6370         hist_enable_trigger(data, rec, event);
6371 }
6372
6373 static struct event_trigger_ops hist_enable_trigger_ops = {
6374         .func                   = hist_enable_trigger,
6375         .print                  = event_enable_trigger_print,
6376         .init                   = event_trigger_init,
6377         .free                   = event_enable_trigger_free,
6378 };
6379
6380 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6381         .func                   = hist_enable_count_trigger,
6382         .print                  = event_enable_trigger_print,
6383         .init                   = event_trigger_init,
6384         .free                   = event_enable_trigger_free,
6385 };
6386
6387 static struct event_trigger_ops hist_disable_trigger_ops = {
6388         .func                   = hist_enable_trigger,
6389         .print                  = event_enable_trigger_print,
6390         .init                   = event_trigger_init,
6391         .free                   = event_enable_trigger_free,
6392 };
6393
6394 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6395         .func                   = hist_enable_count_trigger,
6396         .print                  = event_enable_trigger_print,
6397         .init                   = event_trigger_init,
6398         .free                   = event_enable_trigger_free,
6399 };
6400
6401 static struct event_trigger_ops *
6402 hist_enable_get_trigger_ops(char *cmd, char *param)
6403 {
6404         struct event_trigger_ops *ops;
6405         bool enable;
6406
6407         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6408
6409         if (enable)
6410                 ops = param ? &hist_enable_count_trigger_ops :
6411                         &hist_enable_trigger_ops;
6412         else
6413                 ops = param ? &hist_disable_count_trigger_ops :
6414                         &hist_disable_trigger_ops;
6415
6416         return ops;
6417 }
6418
6419 static void hist_enable_unreg_all(struct trace_event_file *file)
6420 {
6421         struct event_trigger_data *test, *n;
6422
6423         list_for_each_entry_safe(test, n, &file->triggers, list) {
6424                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6425                         list_del_rcu(&test->list);
6426                         update_cond_flag(file);
6427                         trace_event_trigger_enable_disable(file, 0);
6428                         if (test->ops->free)
6429                                 test->ops->free(test->ops, test);
6430                 }
6431         }
6432 }
6433
6434 static struct event_command trigger_hist_enable_cmd = {
6435         .name                   = ENABLE_HIST_STR,
6436         .trigger_type           = ETT_HIST_ENABLE,
6437         .func                   = event_enable_trigger_func,
6438         .reg                    = event_enable_register_trigger,
6439         .unreg                  = event_enable_unregister_trigger,
6440         .unreg_all              = hist_enable_unreg_all,
6441         .get_trigger_ops        = hist_enable_get_trigger_ops,
6442         .set_filter             = set_trigger_filter,
6443 };
6444
6445 static struct event_command trigger_hist_disable_cmd = {
6446         .name                   = DISABLE_HIST_STR,
6447         .trigger_type           = ETT_HIST_ENABLE,
6448         .func                   = event_enable_trigger_func,
6449         .reg                    = event_enable_register_trigger,
6450         .unreg                  = event_enable_unregister_trigger,
6451         .unreg_all              = hist_enable_unreg_all,
6452         .get_trigger_ops        = hist_enable_get_trigger_ops,
6453         .set_filter             = set_trigger_filter,
6454 };
6455
6456 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6457 {
6458         unregister_event_command(&trigger_hist_enable_cmd);
6459         unregister_event_command(&trigger_hist_disable_cmd);
6460 }
6461
6462 __init int register_trigger_hist_enable_disable_cmds(void)
6463 {
6464         int ret;
6465
6466         ret = register_event_command(&trigger_hist_enable_cmd);
6467         if (WARN_ON(ret < 0))
6468                 return ret;
6469         ret = register_event_command(&trigger_hist_disable_cmd);
6470         if (WARN_ON(ret < 0))
6471                 unregister_trigger_hist_enable_disable_cmds();
6472
6473         return ret;
6474 }
6475
6476 static __init int trace_events_hist_init(void)
6477 {
6478         struct dentry *entry = NULL;
6479         struct dentry *d_tracer;
6480         int err = 0;
6481
6482         err = dyn_event_register(&synth_event_ops);
6483         if (err) {
6484                 pr_warn("Could not register synth_event_ops\n");
6485                 return err;
6486         }
6487
6488         d_tracer = tracing_init_dentry();
6489         if (IS_ERR(d_tracer)) {
6490                 err = PTR_ERR(d_tracer);
6491                 goto err;
6492         }
6493
6494         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
6495                                     NULL, &synth_events_fops);
6496         if (!entry) {
6497                 err = -ENODEV;
6498                 goto err;
6499         }
6500
6501         return err;
6502  err:
6503         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6504
6505         return err;
6506 }
6507
6508 fs_initcall(trace_events_hist_init);