linux/kernel/trace/trace_events_hist.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * trace_events_hist - trace event hist triggers
   4 *
   5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6 */
   7
   8#include <linux/module.h>
   9#include <linux/kallsyms.h>
  10#include <linux/security.h>
  11#include <linux/mutex.h>
  12#include <linux/slab.h>
  13#include <linux/stacktrace.h>
  14#include <linux/rculist.h>
  15#include <linux/tracefs.h>
  16
  17/* for gfp flag names */
  18#include <linux/trace_events.h>
  19#include <trace/events/mmflags.h>
  20
  21#include "tracing_map.h"
  22#include "trace.h"
  23#include "trace_dynevent.h"
  24
  25#define SYNTH_SYSTEM            "synthetic"
  26#define SYNTH_FIELDS_MAX        16
  27
  28#define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  29
  30#define ERRORS                                                          \
  31        C(NONE,                 "No error"),                            \
  32        C(DUPLICATE_VAR,        "Variable already defined"),            \
  33        C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  34        C(TOO_MANY_VARS,        "Too many variables defined"),          \
  35        C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  36        C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  37        C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  38        C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  39        C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  40        C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  41        C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  42        C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  43        C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  44        C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  45        C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  46        C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  47        C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  48        C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  49        C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  50        C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  51        C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  52        C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  53        C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  54        C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  55        C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  56        C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  57        C(TOO_MANY_PARAMS,      "Too many action params"),              \
  58        C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  59        C(INVALID_PARAM,        "Invalid action param"),                \
  60        C(ACTION_NOT_FOUND,     "No action found"),                     \
  61        C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  62        C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  63        C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  64        C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  65        C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  66        C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  67        C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  68        C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  69        C(FIELD_NOT_FOUND,      "Couldn't find field"),
  70
  71#undef C
  72#define C(a, b)         HIST_ERR_##a
  73
  74enum { ERRORS };
  75
  76#undef C
  77#define C(a, b)         b
  78
  79static const char *err_text[] = { ERRORS };
  80
  81struct hist_field;
  82
  83typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  84                                struct tracing_map_elt *elt,
  85                                struct ring_buffer_event *rbe,
  86                                void *event);
  87
  88#define HIST_FIELD_OPERANDS_MAX 2
  89#define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  90#define HIST_ACTIONS_MAX        8
  91
  92enum field_op_id {
  93        FIELD_OP_NONE,
  94        FIELD_OP_PLUS,
  95        FIELD_OP_MINUS,
  96        FIELD_OP_UNARY_MINUS,
  97};
  98
  99/*
 100 * A hist_var (histogram variable) contains variable information for
 101 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 102 * flag set.  A hist_var has a variable name e.g. ts0, and is
 103 * associated with a given histogram trigger, as specified by
 104 * hist_data.  The hist_var idx is the unique index assigned to the
 105 * variable by the hist trigger's tracing_map.  The idx is what is
 106 * used to set a variable's value and, by a variable reference, to
 107 * retrieve it.
 108 */
 109struct hist_var {
 110        char                            *name;
 111        struct hist_trigger_data        *hist_data;
 112        unsigned int                    idx;
 113};
 114
 115struct hist_field {
 116        struct ftrace_event_field       *field;
 117        unsigned long                   flags;
 118        hist_field_fn_t                 fn;
 119        unsigned int                    size;
 120        unsigned int                    offset;
 121        unsigned int                    is_signed;
 122        const char                      *type;
 123        struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 124        struct hist_trigger_data        *hist_data;
 125
 126        /*
 127         * Variable fields contain variable-specific info in var.
 128         */
 129        struct hist_var                 var;
 130        enum field_op_id                operator;
 131        char                            *system;
 132        char                            *event_name;
 133
 134        /*
 135         * The name field is used for EXPR and VAR_REF fields.  VAR
 136         * fields contain the variable name in var.name.
 137         */
 138        char                            *name;
 139
 140        /*
 141         * When a histogram trigger is hit, if it has any references
 142         * to variables, the values of those variables are collected
 143         * into a var_ref_vals array by resolve_var_refs().  The
 144         * current value of each variable is read from the tracing_map
 145         * using the hist field's hist_var.idx and entered into the
 146         * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 147         */
 148        unsigned int                    var_ref_idx;
 149        bool                            read_once;
 150};
 151
 152static u64 hist_field_none(struct hist_field *field,
 153                           struct tracing_map_elt *elt,
 154                           struct ring_buffer_event *rbe,
 155                           void *event)
 156{
 157        return 0;
 158}
 159
 160static u64 hist_field_counter(struct hist_field *field,
 161                              struct tracing_map_elt *elt,
 162                              struct ring_buffer_event *rbe,
 163                              void *event)
 164{
 165        return 1;
 166}
 167
 168static u64 hist_field_string(struct hist_field *hist_field,
 169                             struct tracing_map_elt *elt,
 170                             struct ring_buffer_event *rbe,
 171                             void *event)
 172{
 173        char *addr = (char *)(event + hist_field->field->offset);
 174
 175        return (u64)(unsigned long)addr;
 176}
 177
 178static u64 hist_field_dynstring(struct hist_field *hist_field,
 179                                struct tracing_map_elt *elt,
 180                                struct ring_buffer_event *rbe,
 181                                void *event)
 182{
 183        u32 str_item = *(u32 *)(event + hist_field->field->offset);
 184        int str_loc = str_item & 0xffff;
 185        char *addr = (char *)(event + str_loc);
 186
 187        return (u64)(unsigned long)addr;
 188}
 189
 190static u64 hist_field_pstring(struct hist_field *hist_field,
 191                              struct tracing_map_elt *elt,
 192                              struct ring_buffer_event *rbe,
 193                              void *event)
 194{
 195        char **addr = (char **)(event + hist_field->field->offset);
 196
 197        return (u64)(unsigned long)*addr;
 198}
 199
 200static u64 hist_field_log2(struct hist_field *hist_field,
 201                           struct tracing_map_elt *elt,
 202                           struct ring_buffer_event *rbe,
 203                           void *event)
 204{
 205        struct hist_field *operand = hist_field->operands[0];
 206
 207        u64 val = operand->fn(operand, elt, rbe, event);
 208
 209        return (u64) ilog2(roundup_pow_of_two(val));
 210}
 211
 212static u64 hist_field_plus(struct hist_field *hist_field,
 213                           struct tracing_map_elt *elt,
 214                           struct ring_buffer_event *rbe,
 215                           void *event)
 216{
 217        struct hist_field *operand1 = hist_field->operands[0];
 218        struct hist_field *operand2 = hist_field->operands[1];
 219
 220        u64 val1 = operand1->fn(operand1, elt, rbe, event);
 221        u64 val2 = operand2->fn(operand2, elt, rbe, event);
 222
 223        return val1 + val2;
 224}
 225
 226static u64 hist_field_minus(struct hist_field *hist_field,
 227                            struct tracing_map_elt *elt,
 228                            struct ring_buffer_event *rbe,
 229                            void *event)
 230{
 231        struct hist_field *operand1 = hist_field->operands[0];
 232        struct hist_field *operand2 = hist_field->operands[1];
 233
 234        u64 val1 = operand1->fn(operand1, elt, rbe, event);
 235        u64 val2 = operand2->fn(operand2, elt, rbe, event);
 236
 237        return val1 - val2;
 238}
 239
 240static u64 hist_field_unary_minus(struct hist_field *hist_field,
 241                                  struct tracing_map_elt *elt,
 242                                  struct ring_buffer_event *rbe,
 243                                  void *event)
 244{
 245        struct hist_field *operand = hist_field->operands[0];
 246
 247        s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 248        u64 val = (u64)-sval;
 249
 250        return val;
 251}
 252
 253#define DEFINE_HIST_FIELD_FN(type)                                      \
 254        static u64 hist_field_##type(struct hist_field *hist_field,     \
 255                                     struct tracing_map_elt *elt,       \
 256                                     struct ring_buffer_event *rbe,     \
 257                                     void *event)                       \
 258{                                                                       \
 259        type *addr = (type *)(event + hist_field->field->offset);       \
 260                                                                        \
 261        return (u64)(unsigned long)*addr;                               \
 262}
 263
 264DEFINE_HIST_FIELD_FN(s64);
 265DEFINE_HIST_FIELD_FN(u64);
 266DEFINE_HIST_FIELD_FN(s32);
 267DEFINE_HIST_FIELD_FN(u32);
 268DEFINE_HIST_FIELD_FN(s16);
 269DEFINE_HIST_FIELD_FN(u16);
 270DEFINE_HIST_FIELD_FN(s8);
 271DEFINE_HIST_FIELD_FN(u8);
 272
 273#define for_each_hist_field(i, hist_data)       \
 274        for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 275
 276#define for_each_hist_val_field(i, hist_data)   \
 277        for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 278
 279#define for_each_hist_key_field(i, hist_data)   \
 280        for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 281
 282#define HIST_STACKTRACE_DEPTH   16
 283#define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 284#define HIST_STACKTRACE_SKIP    5
 285
 286#define HITCOUNT_IDX            0
 287#define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 288
 289enum hist_field_flags {
 290        HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 291        HIST_FIELD_FL_KEY               = 1 << 1,
 292        HIST_FIELD_FL_STRING            = 1 << 2,
 293        HIST_FIELD_FL_HEX               = 1 << 3,
 294        HIST_FIELD_FL_SYM               = 1 << 4,
 295        HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 296        HIST_FIELD_FL_EXECNAME          = 1 << 6,
 297        HIST_FIELD_FL_SYSCALL           = 1 << 7,
 298        HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 299        HIST_FIELD_FL_LOG2              = 1 << 9,
 300        HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 301        HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 302        HIST_FIELD_FL_VAR               = 1 << 12,
 303        HIST_FIELD_FL_EXPR              = 1 << 13,
 304        HIST_FIELD_FL_VAR_REF           = 1 << 14,
 305        HIST_FIELD_FL_CPU               = 1 << 15,
 306        HIST_FIELD_FL_ALIAS             = 1 << 16,
 307};
 308
 309struct var_defs {
 310        unsigned int    n_vars;
 311        char            *name[TRACING_MAP_VARS_MAX];
 312        char            *expr[TRACING_MAP_VARS_MAX];
 313};
 314
 315struct hist_trigger_attrs {
 316        char            *keys_str;
 317        char            *vals_str;
 318        char            *sort_key_str;
 319        char            *name;
 320        char            *clock;
 321        bool            pause;
 322        bool            cont;
 323        bool            clear;
 324        bool            ts_in_usecs;
 325        unsigned int    map_bits;
 326
 327        char            *assignment_str[TRACING_MAP_VARS_MAX];
 328        unsigned int    n_assignments;
 329
 330        char            *action_str[HIST_ACTIONS_MAX];
 331        unsigned int    n_actions;
 332
 333        struct var_defs var_defs;
 334};
 335
 336struct field_var {
 337        struct hist_field       *var;
 338        struct hist_field       *val;
 339};
 340
 341struct field_var_hist {
 342        struct hist_trigger_data        *hist_data;
 343        char                            *cmd;
 344};
 345
 346struct hist_trigger_data {
 347        struct hist_field               *fields[HIST_FIELDS_MAX];
 348        unsigned int                    n_vals;
 349        unsigned int                    n_keys;
 350        unsigned int                    n_fields;
 351        unsigned int                    n_vars;
 352        unsigned int                    key_size;
 353        struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 354        unsigned int                    n_sort_keys;
 355        struct trace_event_file         *event_file;
 356        struct hist_trigger_attrs       *attrs;
 357        struct tracing_map              *map;
 358        bool                            enable_timestamps;
 359        bool                            remove;
 360        struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 361        unsigned int                    n_var_refs;
 362
 363        struct action_data              *actions[HIST_ACTIONS_MAX];
 364        unsigned int                    n_actions;
 365
 366        struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 367        unsigned int                    n_field_vars;
 368        unsigned int                    n_field_var_str;
 369        struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 370        unsigned int                    n_field_var_hists;
 371
 372        struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 373        unsigned int                    n_save_vars;
 374        unsigned int                    n_save_var_str;
 375};
 376
 377static int synth_event_create(int argc, const char **argv);
 378static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 379static int synth_event_release(struct dyn_event *ev);
 380static bool synth_event_is_busy(struct dyn_event *ev);
 381static bool synth_event_match(const char *system, const char *event,
 382                        int argc, const char **argv, struct dyn_event *ev);
 383
 384static struct dyn_event_operations synth_event_ops = {
 385        .create = synth_event_create,
 386        .show = synth_event_show,
 387        .is_busy = synth_event_is_busy,
 388        .free = synth_event_release,
 389        .match = synth_event_match,
 390};
 391
 392struct synth_field {
 393        char *type;
 394        char *name;
 395        size_t size;
 396        bool is_signed;
 397        bool is_string;
 398};
 399
 400struct synth_event {
 401        struct dyn_event                        devent;
 402        int                                     ref;
 403        char                                    *name;
 404        struct synth_field                      **fields;
 405        unsigned int                            n_fields;
 406        unsigned int                            n_u64;
 407        struct trace_event_class                class;
 408        struct trace_event_call                 call;
 409        struct tracepoint                       *tp;
 410};
 411
 412static bool is_synth_event(struct dyn_event *ev)
 413{
 414        return ev->ops == &synth_event_ops;
 415}
 416
 417static struct synth_event *to_synth_event(struct dyn_event *ev)
 418{
 419        return container_of(ev, struct synth_event, devent);
 420}
 421
 422static bool synth_event_is_busy(struct dyn_event *ev)
 423{
 424        struct synth_event *event = to_synth_event(ev);
 425
 426        return event->ref != 0;
 427}
 428
 429static bool synth_event_match(const char *system, const char *event,
 430                        int argc, const char **argv, struct dyn_event *ev)
 431{
 432        struct synth_event *sev = to_synth_event(ev);
 433
 434        return strcmp(sev->name, event) == 0 &&
 435                (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 436}
 437
 438struct action_data;
 439
 440typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 441                             struct tracing_map_elt *elt, void *rec,
 442                             struct ring_buffer_event *rbe, void *key,
 443                             struct action_data *data, u64 *var_ref_vals);
 444
 445typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 446
 447enum handler_id {
 448        HANDLER_ONMATCH = 1,
 449        HANDLER_ONMAX,
 450        HANDLER_ONCHANGE,
 451};
 452
 453enum action_id {
 454        ACTION_SAVE = 1,
 455        ACTION_TRACE,
 456        ACTION_SNAPSHOT,
 457};
 458
 459struct action_data {
 460        enum handler_id         handler;
 461        enum action_id          action;
 462        char                    *action_name;
 463        action_fn_t             fn;
 464
 465        unsigned int            n_params;
 466        char                    *params[SYNTH_FIELDS_MAX];
 467
 468        /*
 469         * When a histogram trigger is hit, the values of any
 470         * references to variables, including variables being passed
 471         * as parameters to synthetic events, are collected into a
 472         * var_ref_vals array.  This var_ref_idx is the index of the
 473         * first param in the array to be passed to the synthetic
 474         * event invocation.
 475         */
 476        unsigned int            var_ref_idx;
 477        struct synth_event      *synth_event;
 478        bool                    use_trace_keyword;
 479        char                    *synth_event_name;
 480
 481        union {
 482                struct {
 483                        char                    *event;
 484                        char                    *event_system;
 485                } match_data;
 486
 487                struct {
 488                        /*
 489                         * var_str contains the $-unstripped variable
 490                         * name referenced by var_ref, and used when
 491                         * printing the action.  Because var_ref
 492                         * creation is deferred to create_actions(),
 493                         * we need a per-action way to save it until
 494                         * then, thus var_str.
 495                         */
 496                        char                    *var_str;
 497
 498                        /*
 499                         * var_ref refers to the variable being
 500                         * tracked e.g onmax($var).
 501                         */
 502                        struct hist_field       *var_ref;
 503
 504                        /*
 505                         * track_var contains the 'invisible' tracking
 506                         * variable created to keep the current
 507                         * e.g. max value.
 508                         */
 509                        struct hist_field       *track_var;
 510
 511                        check_track_val_fn_t    check_val;
 512                        action_fn_t             save_data;
 513                } track_data;
 514        };
 515};
 516
 517struct track_data {
 518        u64                             track_val;
 519        bool                            updated;
 520
 521        unsigned int                    key_len;
 522        void                            *key;
 523        struct tracing_map_elt          elt;
 524
 525        struct action_data              *action_data;
 526        struct hist_trigger_data        *hist_data;
 527};
 528
 529struct hist_elt_data {
 530        char *comm;
 531        u64 *var_ref_vals;
 532        char *field_var_str[SYNTH_FIELDS_MAX];
 533};
 534
 535struct snapshot_context {
 536        struct tracing_map_elt  *elt;
 537        void                    *key;
 538};
 539
 540static void track_data_free(struct track_data *track_data)
 541{
 542        struct hist_elt_data *elt_data;
 543
 544        if (!track_data)
 545                return;
 546
 547        kfree(track_data->key);
 548
 549        elt_data = track_data->elt.private_data;
 550        if (elt_data) {
 551                kfree(elt_data->comm);
 552                kfree(elt_data);
 553        }
 554
 555        kfree(track_data);
 556}
 557
 558static struct track_data *track_data_alloc(unsigned int key_len,
 559                                           struct action_data *action_data,
 560                                           struct hist_trigger_data *hist_data)
 561{
 562        struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 563        struct hist_elt_data *elt_data;
 564
 565        if (!data)
 566                return ERR_PTR(-ENOMEM);
 567
 568        data->key = kzalloc(key_len, GFP_KERNEL);
 569        if (!data->key) {
 570                track_data_free(data);
 571                return ERR_PTR(-ENOMEM);
 572        }
 573
 574        data->key_len = key_len;
 575        data->action_data = action_data;
 576        data->hist_data = hist_data;
 577
 578        elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 579        if (!elt_data) {
 580                track_data_free(data);
 581                return ERR_PTR(-ENOMEM);
 582        }
 583        data->elt.private_data = elt_data;
 584
 585        elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 586        if (!elt_data->comm) {
 587                track_data_free(data);
 588                return ERR_PTR(-ENOMEM);
 589        }
 590
 591        return data;
 592}
 593
 594static char last_cmd[MAX_FILTER_STR_VAL];
 595static char last_cmd_loc[MAX_FILTER_STR_VAL];
 596
 597static int errpos(char *str)
 598{
 599        return err_pos(last_cmd, str);
 600}
 601
 602static void last_cmd_set(struct trace_event_file *file, char *str)
 603{
 604        const char *system = NULL, *name = NULL;
 605        struct trace_event_call *call;
 606
 607        if (!str)
 608                return;
 609
 610        strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1);
 611
 612        if (file) {
 613                call = file->event_call;
 614
 615                system = call->class->system;
 616                if (system) {
 617                        name = trace_event_name(call);
 618                        if (!name)
 619                                system = NULL;
 620                }
 621        }
 622
 623        if (system)
 624                snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 625}
 626
 627static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 628{
 629        tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 630                        err_type, err_pos);
 631}
 632
 633static void hist_err_clear(void)
 634{
 635        last_cmd[0] = '\0';
 636        last_cmd_loc[0] = '\0';
 637}
 638
 639struct synth_trace_event {
 640        struct trace_entry      ent;
 641        u64                     fields[];
 642};
 643
 644static int synth_event_define_fields(struct trace_event_call *call)
 645{
 646        struct synth_trace_event trace;
 647        int offset = offsetof(typeof(trace), fields);
 648        struct synth_event *event = call->data;
 649        unsigned int i, size, n_u64;
 650        char *name, *type;
 651        bool is_signed;
 652        int ret = 0;
 653
 654        for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 655                size = event->fields[i]->size;
 656                is_signed = event->fields[i]->is_signed;
 657                type = event->fields[i]->type;
 658                name = event->fields[i]->name;
 659                ret = trace_define_field(call, type, name, offset, size,
 660                                         is_signed, FILTER_OTHER);
 661                if (ret)
 662                        break;
 663
 664                if (event->fields[i]->is_string) {
 665                        offset += STR_VAR_LEN_MAX;
 666                        n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 667                } else {
 668                        offset += sizeof(u64);
 669                        n_u64++;
 670                }
 671        }
 672
 673        event->n_u64 = n_u64;
 674
 675        return ret;
 676}
 677
 678static bool synth_field_signed(char *type)
 679{
 680        if (str_has_prefix(type, "u"))
 681                return false;
 682        if (strcmp(type, "gfp_t") == 0)
 683                return false;
 684
 685        return true;
 686}
 687
 688static int synth_field_is_string(char *type)
 689{
 690        if (strstr(type, "char[") != NULL)
 691                return true;
 692
 693        return false;
 694}
 695
 696static int synth_field_string_size(char *type)
 697{
 698        char buf[4], *end, *start;
 699        unsigned int len;
 700        int size, err;
 701
 702        start = strstr(type, "char[");
 703        if (start == NULL)
 704                return -EINVAL;
 705        start += sizeof("char[") - 1;
 706
 707        end = strchr(type, ']');
 708        if (!end || end < start)
 709                return -EINVAL;
 710
 711        len = end - start;
 712        if (len > 3)
 713                return -EINVAL;
 714
 715        strncpy(buf, start, len);
 716        buf[len] = '\0';
 717
 718        err = kstrtouint(buf, 0, &size);
 719        if (err)
 720                return err;
 721
 722        if (size > STR_VAR_LEN_MAX)
 723                return -EINVAL;
 724
 725        return size;
 726}
 727
 728static int synth_field_size(char *type)
 729{
 730        int size = 0;
 731
 732        if (strcmp(type, "s64") == 0)
 733                size = sizeof(s64);
 734        else if (strcmp(type, "u64") == 0)
 735                size = sizeof(u64);
 736        else if (strcmp(type, "s32") == 0)
 737                size = sizeof(s32);
 738        else if (strcmp(type, "u32") == 0)
 739                size = sizeof(u32);
 740        else if (strcmp(type, "s16") == 0)
 741                size = sizeof(s16);
 742        else if (strcmp(type, "u16") == 0)
 743                size = sizeof(u16);
 744        else if (strcmp(type, "s8") == 0)
 745                size = sizeof(s8);
 746        else if (strcmp(type, "u8") == 0)
 747                size = sizeof(u8);
 748        else if (strcmp(type, "char") == 0)
 749                size = sizeof(char);
 750        else if (strcmp(type, "unsigned char") == 0)
 751                size = sizeof(unsigned char);
 752        else if (strcmp(type, "int") == 0)
 753                size = sizeof(int);
 754        else if (strcmp(type, "unsigned int") == 0)
 755                size = sizeof(unsigned int);
 756        else if (strcmp(type, "long") == 0)
 757                size = sizeof(long);
 758        else if (strcmp(type, "unsigned long") == 0)
 759                size = sizeof(unsigned long);
 760        else if (strcmp(type, "pid_t") == 0)
 761                size = sizeof(pid_t);
 762        else if (strcmp(type, "gfp_t") == 0)
 763                size = sizeof(gfp_t);
 764        else if (synth_field_is_string(type))
 765                size = synth_field_string_size(type);
 766
 767        return size;
 768}
 769
 770static const char *synth_field_fmt(char *type)
 771{
 772        const char *fmt = "%llu";
 773
 774        if (strcmp(type, "s64") == 0)
 775                fmt = "%lld";
 776        else if (strcmp(type, "u64") == 0)
 777                fmt = "%llu";
 778        else if (strcmp(type, "s32") == 0)
 779                fmt = "%d";
 780        else if (strcmp(type, "u32") == 0)
 781                fmt = "%u";
 782        else if (strcmp(type, "s16") == 0)
 783                fmt = "%d";
 784        else if (strcmp(type, "u16") == 0)
 785                fmt = "%u";
 786        else if (strcmp(type, "s8") == 0)
 787                fmt = "%d";
 788        else if (strcmp(type, "u8") == 0)
 789                fmt = "%u";
 790        else if (strcmp(type, "char") == 0)
 791                fmt = "%d";
 792        else if (strcmp(type, "unsigned char") == 0)
 793                fmt = "%u";
 794        else if (strcmp(type, "int") == 0)
 795                fmt = "%d";
 796        else if (strcmp(type, "unsigned int") == 0)
 797                fmt = "%u";
 798        else if (strcmp(type, "long") == 0)
 799                fmt = "%ld";
 800        else if (strcmp(type, "unsigned long") == 0)
 801                fmt = "%lu";
 802        else if (strcmp(type, "pid_t") == 0)
 803                fmt = "%d";
 804        else if (strcmp(type, "gfp_t") == 0)
 805                fmt = "%x";
 806        else if (synth_field_is_string(type))
 807                fmt = "%s";
 808
 809        return fmt;
 810}
 811
 812static enum print_line_t print_synth_event(struct trace_iterator *iter,
 813                                           int flags,
 814                                           struct trace_event *event)
 815{
 816        struct trace_array *tr = iter->tr;
 817        struct trace_seq *s = &iter->seq;
 818        struct synth_trace_event *entry;
 819        struct synth_event *se;
 820        unsigned int i, n_u64;
 821        char print_fmt[32];
 822        const char *fmt;
 823
 824        entry = (struct synth_trace_event *)iter->ent;
 825        se = container_of(event, struct synth_event, call.event);
 826
 827        trace_seq_printf(s, "%s: ", se->name);
 828
 829        for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 830                if (trace_seq_has_overflowed(s))
 831                        goto end;
 832
 833                fmt = synth_field_fmt(se->fields[i]->type);
 834
 835                /* parameter types */
 836                if (tr->trace_flags & TRACE_ITER_VERBOSE)
 837                        trace_seq_printf(s, "%s ", fmt);
 838
 839                snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 840
 841                /* parameter values */
 842                if (se->fields[i]->is_string) {
 843                        trace_seq_printf(s, print_fmt, se->fields[i]->name,
 844                                         (char *)&entry->fields[n_u64],
 845                                         i == se->n_fields - 1 ? "" : " ");
 846                        n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 847                } else {
 848                        struct trace_print_flags __flags[] = {
 849                            __def_gfpflag_names, {-1, NULL} };
 850
 851                        trace_seq_printf(s, print_fmt, se->fields[i]->name,
 852                                         entry->fields[n_u64],
 853                                         i == se->n_fields - 1 ? "" : " ");
 854
 855                        if (strcmp(se->fields[i]->type, "gfp_t") == 0) {
 856                                trace_seq_puts(s, " (");
 857                                trace_print_flags_seq(s, "|",
 858                                                      entry->fields[n_u64],
 859                                                      __flags);
 860                                trace_seq_putc(s, ')');
 861                        }
 862                        n_u64++;
 863                }
 864        }
 865end:
 866        trace_seq_putc(s, '\n');
 867
 868        return trace_handle_return(s);
 869}
 870
 871static struct trace_event_functions synth_event_funcs = {
 872        .trace          = print_synth_event
 873};
 874
 875static notrace void trace_event_raw_event_synth(void *__data,
 876                                                u64 *var_ref_vals,
 877                                                unsigned int var_ref_idx)
 878{
 879        struct trace_event_file *trace_file = __data;
 880        struct synth_trace_event *entry;
 881        struct trace_event_buffer fbuffer;
 882        struct ring_buffer *buffer;
 883        struct synth_event *event;
 884        unsigned int i, n_u64;
 885        int fields_size = 0;
 886
 887        event = trace_file->event_call->data;
 888
 889        if (trace_trigger_soft_disabled(trace_file))
 890                return;
 891
 892        fields_size = event->n_u64 * sizeof(u64);
 893
 894        /*
 895         * Avoid ring buffer recursion detection, as this event
 896         * is being performed within another event.
 897         */
 898        buffer = trace_file->tr->trace_buffer.buffer;
 899        ring_buffer_nest_start(buffer);
 900
 901        entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 902                                           sizeof(*entry) + fields_size);
 903        if (!entry)
 904                goto out;
 905
 906        for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 907                if (event->fields[i]->is_string) {
 908                        char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 909                        char *str_field = (char *)&entry->fields[n_u64];
 910
 911                        strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 912                        n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 913                } else {
 914                        entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 915                        n_u64++;
 916                }
 917        }
 918
 919        trace_event_buffer_commit(&fbuffer);
 920out:
 921        ring_buffer_nest_end(buffer);
 922}
 923
 924static void free_synth_event_print_fmt(struct trace_event_call *call)
 925{
 926        if (call) {
 927                kfree(call->print_fmt);
 928                call->print_fmt = NULL;
 929        }
 930}
 931
 932static int __set_synth_event_print_fmt(struct synth_event *event,
 933                                       char *buf, int len)
 934{
 935        const char *fmt;
 936        int pos = 0;
 937        int i;
 938
 939        /* When len=0, we just calculate the needed length */
 940#define LEN_OR_ZERO (len ? len - pos : 0)
 941
 942        pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 943        for (i = 0; i < event->n_fields; i++) {
 944                fmt = synth_field_fmt(event->fields[i]->type);
 945                pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 946                                event->fields[i]->name, fmt,
 947                                i == event->n_fields - 1 ? "" : ", ");
 948        }
 949        pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 950
 951        for (i = 0; i < event->n_fields; i++) {
 952                pos += snprintf(buf + pos, LEN_OR_ZERO,
 953                                ", REC->%s", event->fields[i]->name);
 954        }
 955
 956#undef LEN_OR_ZERO
 957
 958        /* return the length of print_fmt */
 959        return pos;
 960}
 961
 962static int set_synth_event_print_fmt(struct trace_event_call *call)
 963{
 964        struct synth_event *event = call->data;
 965        char *print_fmt;
 966        int len;
 967
 968        /* First: called with 0 length to calculate the needed length */
 969        len = __set_synth_event_print_fmt(event, NULL, 0);
 970
 971        print_fmt = kmalloc(len + 1, GFP_KERNEL);
 972        if (!print_fmt)
 973                return -ENOMEM;
 974
 975        /* Second: actually write the @print_fmt */
 976        __set_synth_event_print_fmt(event, print_fmt, len + 1);
 977        call->print_fmt = print_fmt;
 978
 979        return 0;
 980}
 981
 982static void free_synth_field(struct synth_field *field)
 983{
 984        kfree(field->type);
 985        kfree(field->name);
 986        kfree(field);
 987}
 988
 989static struct synth_field *parse_synth_field(int argc, const char **argv,
 990                                             int *consumed)
 991{
 992        struct synth_field *field;
 993        const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
 994        int len, ret = 0;
 995
 996        if (field_type[0] == ';')
 997                field_type++;
 998
 999        if (!strcmp(field_type, "unsigned")) {
1000                if (argc < 3)
1001                        return ERR_PTR(-EINVAL);
1002                prefix = "unsigned ";
1003                field_type = argv[1];
1004                field_name = argv[2];
1005                *consumed = 3;
1006        } else {
1007                field_name = argv[1];
1008                *consumed = 2;
1009        }
1010
1011        field = kzalloc(sizeof(*field), GFP_KERNEL);
1012        if (!field)
1013                return ERR_PTR(-ENOMEM);
1014
1015        len = strlen(field_name);
1016        array = strchr(field_name, '[');
1017        if (array)
1018                len -= strlen(array);
1019        else if (field_name[len - 1] == ';')
1020                len--;
1021
1022        field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1023        if (!field->name) {
1024                ret = -ENOMEM;
1025                goto free;
1026        }
1027
1028        if (field_type[0] == ';')
1029                field_type++;
1030        len = strlen(field_type) + 1;
1031        if (array)
1032                len += strlen(array);
1033        if (prefix)
1034                len += strlen(prefix);
1035
1036        field->type = kzalloc(len, GFP_KERNEL);
1037        if (!field->type) {
1038                ret = -ENOMEM;
1039                goto free;
1040        }
1041        if (prefix)
1042                strcat(field->type, prefix);
1043        strcat(field->type, field_type);
1044        if (array) {
1045                strcat(field->type, array);
1046                if (field->type[len - 1] == ';')
1047                        field->type[len - 1] = '\0';
1048        }
1049
1050        field->size = synth_field_size(field->type);
1051        if (!field->size) {
1052                ret = -EINVAL;
1053                goto free;
1054        }
1055
1056        if (synth_field_is_string(field->type))
1057                field->is_string = true;
1058
1059        field->is_signed = synth_field_signed(field->type);
1060
1061 out:
1062        return field;
1063 free:
1064        free_synth_field(field);
1065        field = ERR_PTR(ret);
1066        goto out;
1067}
1068
1069static void free_synth_tracepoint(struct tracepoint *tp)
1070{
1071        if (!tp)
1072                return;
1073
1074        kfree(tp->name);
1075        kfree(tp);
1076}
1077
1078static struct tracepoint *alloc_synth_tracepoint(char *name)
1079{
1080        struct tracepoint *tp;
1081
1082        tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1083        if (!tp)
1084                return ERR_PTR(-ENOMEM);
1085
1086        tp->name = kstrdup(name, GFP_KERNEL);
1087        if (!tp->name) {
1088                kfree(tp);
1089                return ERR_PTR(-ENOMEM);
1090        }
1091
1092        return tp;
1093}
1094
1095typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1096                                    unsigned int var_ref_idx);
1097
1098static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1099                               unsigned int var_ref_idx)
1100{
1101        struct tracepoint *tp = event->tp;
1102
1103        if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1104                struct tracepoint_func *probe_func_ptr;
1105                synth_probe_func_t probe_func;
1106                void *__data;
1107
1108                if (!(cpu_online(raw_smp_processor_id())))
1109                        return;
1110
1111                probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1112                if (probe_func_ptr) {
1113                        do {
1114                                probe_func = probe_func_ptr->func;
1115                                __data = probe_func_ptr->data;
1116                                probe_func(__data, var_ref_vals, var_ref_idx);
1117                        } while ((++probe_func_ptr)->func);
1118                }
1119        }
1120}
1121
1122static struct synth_event *find_synth_event(const char *name)
1123{
1124        struct dyn_event *pos;
1125        struct synth_event *event;
1126
1127        for_each_dyn_event(pos) {
1128                if (!is_synth_event(pos))
1129                        continue;
1130                event = to_synth_event(pos);
1131                if (strcmp(event->name, name) == 0)
1132                        return event;
1133        }
1134
1135        return NULL;
1136}
1137
1138static int register_synth_event(struct synth_event *event)
1139{
1140        struct trace_event_call *call = &event->call;
1141        int ret = 0;
1142
1143        event->call.class = &event->class;
1144        event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1145        if (!event->class.system) {
1146                ret = -ENOMEM;
1147                goto out;
1148        }
1149
1150        event->tp = alloc_synth_tracepoint(event->name);
1151        if (IS_ERR(event->tp)) {
1152                ret = PTR_ERR(event->tp);
1153                event->tp = NULL;
1154                goto out;
1155        }
1156
1157        INIT_LIST_HEAD(&call->class->fields);
1158        call->event.funcs = &synth_event_funcs;
1159        call->class->define_fields = synth_event_define_fields;
1160
1161        ret = register_trace_event(&call->event);
1162        if (!ret) {
1163                ret = -ENODEV;
1164                goto out;
1165        }
1166        call->flags = TRACE_EVENT_FL_TRACEPOINT;
1167        call->class->reg = trace_event_reg;
1168        call->class->probe = trace_event_raw_event_synth;
1169        call->data = event;
1170        call->tp = event->tp;
1171
1172        ret = trace_add_event_call(call);
1173        if (ret) {
1174                pr_warn("Failed to register synthetic event: %s\n",
1175                        trace_event_name(call));
1176                goto err;
1177        }
1178
1179        ret = set_synth_event_print_fmt(call);
1180        if (ret < 0) {
1181                trace_remove_event_call(call);
1182                goto err;
1183        }
1184 out:
1185        return ret;
1186 err:
1187        unregister_trace_event(&call->event);
1188        goto out;
1189}
1190
1191static int unregister_synth_event(struct synth_event *event)
1192{
1193        struct trace_event_call *call = &event->call;
1194        int ret;
1195
1196        ret = trace_remove_event_call(call);
1197
1198        return ret;
1199}
1200
1201static void free_synth_event(struct synth_event *event)
1202{
1203        unsigned int i;
1204
1205        if (!event)
1206                return;
1207
1208        for (i = 0; i < event->n_fields; i++)
1209                free_synth_field(event->fields[i]);
1210
1211        kfree(event->fields);
1212        kfree(event->name);
1213        kfree(event->class.system);
1214        free_synth_tracepoint(event->tp);
1215        free_synth_event_print_fmt(&event->call);
1216        kfree(event);
1217}
1218
1219static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1220                                             struct synth_field **fields)
1221{
1222        struct synth_event *event;
1223        unsigned int i;
1224
1225        event = kzalloc(sizeof(*event), GFP_KERNEL);
1226        if (!event) {
1227                event = ERR_PTR(-ENOMEM);
1228                goto out;
1229        }
1230
1231        event->name = kstrdup(name, GFP_KERNEL);
1232        if (!event->name) {
1233                kfree(event);
1234                event = ERR_PTR(-ENOMEM);
1235                goto out;
1236        }
1237
1238        event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1239        if (!event->fields) {
1240                free_synth_event(event);
1241                event = ERR_PTR(-ENOMEM);
1242                goto out;
1243        }
1244
1245        dyn_event_init(&event->devent, &synth_event_ops);
1246
1247        for (i = 0; i < n_fields; i++)
1248                event->fields[i] = fields[i];
1249
1250        event->n_fields = n_fields;
1251 out:
1252        return event;
1253}
1254
1255static void action_trace(struct hist_trigger_data *hist_data,
1256                         struct tracing_map_elt *elt, void *rec,
1257                         struct ring_buffer_event *rbe, void *key,
1258                         struct action_data *data, u64 *var_ref_vals)
1259{
1260        struct synth_event *event = data->synth_event;
1261
1262        trace_synth(event, var_ref_vals, data->var_ref_idx);
1263}
1264
1265struct hist_var_data {
1266        struct list_head list;
1267        struct hist_trigger_data *hist_data;
1268};
1269
1270static int __create_synth_event(int argc, const char *name, const char **argv)
1271{
1272        struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1273        struct synth_event *event = NULL;
1274        int i, consumed = 0, n_fields = 0, ret = 0;
1275
1276        /*
1277         * Argument syntax:
1278         *  - Add synthetic event: <event_name> field[;field] ...
1279         *  - Remove synthetic event: !<event_name> field[;field] ...
1280         *      where 'field' = type field_name
1281         */
1282
1283        if (name[0] == '\0' || argc < 1)
1284                return -EINVAL;
1285
1286        mutex_lock(&event_mutex);
1287
1288        event = find_synth_event(name);
1289        if (event) {
1290                ret = -EEXIST;
1291                goto out;
1292        }
1293
1294        for (i = 0; i < argc - 1; i++) {
1295                if (strcmp(argv[i], ";") == 0)
1296                        continue;
1297                if (n_fields == SYNTH_FIELDS_MAX) {
1298                        ret = -EINVAL;
1299                        goto err;
1300                }
1301
1302                field = parse_synth_field(argc - i, &argv[i], &consumed);
1303                if (IS_ERR(field)) {
1304                        ret = PTR_ERR(field);
1305                        goto err;
1306                }
1307                fields[n_fields++] = field;
1308                i += consumed - 1;
1309        }
1310
1311        if (i < argc && strcmp(argv[i], ";") != 0) {
1312                ret = -EINVAL;
1313                goto err;
1314        }
1315
1316        event = alloc_synth_event(name, n_fields, fields);
1317        if (IS_ERR(event)) {
1318                ret = PTR_ERR(event);
1319                event = NULL;
1320                goto err;
1321        }
1322        ret = register_synth_event(event);
1323        if (!ret)
1324                dyn_event_add(&event->devent);
1325        else
1326                free_synth_event(event);
1327 out:
1328        mutex_unlock(&event_mutex);
1329
1330        return ret;
1331 err:
1332        for (i = 0; i < n_fields; i++)
1333                free_synth_field(fields[i]);
1334
1335        goto out;
1336}
1337
1338static int create_or_delete_synth_event(int argc, char **argv)
1339{
1340        const char *name = argv[0];
1341        struct synth_event *event = NULL;
1342        int ret;
1343
1344        /* trace_run_command() ensures argc != 0 */
1345        if (name[0] == '!') {
1346                mutex_lock(&event_mutex);
1347                event = find_synth_event(name + 1);
1348                if (event) {
1349                        if (event->ref)
1350                                ret = -EBUSY;
1351                        else {
1352                                ret = unregister_synth_event(event);
1353                                if (!ret) {
1354                                        dyn_event_remove(&event->devent);
1355                                        free_synth_event(event);
1356                                }
1357                        }
1358                } else
1359                        ret = -ENOENT;
1360                mutex_unlock(&event_mutex);
1361                return ret;
1362        }
1363
1364        ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1365        return ret == -ECANCELED ? -EINVAL : ret;
1366}
1367
1368static int synth_event_create(int argc, const char **argv)
1369{
1370        const char *name = argv[0];
1371        int len;
1372
1373        if (name[0] != 's' || name[1] != ':')
1374                return -ECANCELED;
1375        name += 2;
1376
1377        /* This interface accepts group name prefix */
1378        if (strchr(name, '/')) {
1379                len = str_has_prefix(name, SYNTH_SYSTEM "/");
1380                if (len == 0)
1381                        return -EINVAL;
1382                name += len;
1383        }
1384        return __create_synth_event(argc - 1, name, argv + 1);
1385}
1386
1387static int synth_event_release(struct dyn_event *ev)
1388{
1389        struct synth_event *event = to_synth_event(ev);
1390        int ret;
1391
1392        if (event->ref)
1393                return -EBUSY;
1394
1395        ret = unregister_synth_event(event);
1396        if (ret)
1397                return ret;
1398
1399        dyn_event_remove(ev);
1400        free_synth_event(event);
1401        return 0;
1402}
1403
1404static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1405{
1406        struct synth_field *field;
1407        unsigned int i;
1408
1409        seq_printf(m, "%s\t", event->name);
1410
1411        for (i = 0; i < event->n_fields; i++) {
1412                field = event->fields[i];
1413
1414                /* parameter values */
1415                seq_printf(m, "%s %s%s", field->type, field->name,
1416                           i == event->n_fields - 1 ? "" : "; ");
1417        }
1418
1419        seq_putc(m, '\n');
1420
1421        return 0;
1422}
1423
1424static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1425{
1426        struct synth_event *event = to_synth_event(ev);
1427
1428        seq_printf(m, "s:%s/", event->class.system);
1429
1430        return __synth_event_show(m, event);
1431}
1432
1433static int synth_events_seq_show(struct seq_file *m, void *v)
1434{
1435        struct dyn_event *ev = v;
1436
1437        if (!is_synth_event(ev))
1438                return 0;
1439
1440        return __synth_event_show(m, to_synth_event(ev));
1441}
1442
1443static const struct seq_operations synth_events_seq_op = {
1444        .start  = dyn_event_seq_start,
1445        .next   = dyn_event_seq_next,
1446        .stop   = dyn_event_seq_stop,
1447        .show   = synth_events_seq_show,
1448};
1449
1450static int synth_events_open(struct inode *inode, struct file *file)
1451{
1452        int ret;
1453
1454        ret = security_locked_down(LOCKDOWN_TRACEFS);
1455        if (ret)
1456                return ret;
1457
1458        if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1459                ret = dyn_events_release_all(&synth_event_ops);
1460                if (ret < 0)
1461                        return ret;
1462        }
1463
1464        return seq_open(file, &synth_events_seq_op);
1465}
1466
1467static ssize_t synth_events_write(struct file *file,
1468                                  const char __user *buffer,
1469                                  size_t count, loff_t *ppos)
1470{
1471        return trace_parse_run_command(file, buffer, count, ppos,
1472                                       create_or_delete_synth_event);
1473}
1474
1475static const struct file_operations synth_events_fops = {
1476        .open           = synth_events_open,
1477        .write          = synth_events_write,
1478        .read           = seq_read,
1479        .llseek         = seq_lseek,
1480        .release        = seq_release,
1481};
1482
1483static u64 hist_field_timestamp(struct hist_field *hist_field,
1484                                struct tracing_map_elt *elt,
1485                                struct ring_buffer_event *rbe,
1486                                void *event)
1487{
1488        struct hist_trigger_data *hist_data = hist_field->hist_data;
1489        struct trace_array *tr = hist_data->event_file->tr;
1490
1491        u64 ts = ring_buffer_event_time_stamp(rbe);
1492
1493        if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1494                ts = ns2usecs(ts);
1495
1496        return ts;
1497}
1498
1499static u64 hist_field_cpu(struct hist_field *hist_field,
1500                          struct tracing_map_elt *elt,
1501                          struct ring_buffer_event *rbe,
1502                          void *event)
1503{
1504        int cpu = smp_processor_id();
1505
1506        return cpu;
1507}
1508
1509/**
1510 * check_field_for_var_ref - Check if a VAR_REF field references a variable
1511 * @hist_field: The VAR_REF field to check
1512 * @var_data: The hist trigger that owns the variable
1513 * @var_idx: The trigger variable identifier
1514 *
1515 * Check the given VAR_REF field to see whether or not it references
1516 * the given variable associated with the given trigger.
1517 *
1518 * Return: The VAR_REF field if it does reference the variable, NULL if not
1519 */
1520static struct hist_field *
1521check_field_for_var_ref(struct hist_field *hist_field,
1522                        struct hist_trigger_data *var_data,
1523                        unsigned int var_idx)
1524{
1525        WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1526
1527        if (hist_field && hist_field->var.idx == var_idx &&
1528            hist_field->var.hist_data == var_data)
1529                return hist_field;
1530
1531        return NULL;
1532}
1533
1534/**
1535 * find_var_ref - Check if a trigger has a reference to a trigger variable
1536 * @hist_data: The hist trigger that might have a reference to the variable
1537 * @var_data: The hist trigger that owns the variable
1538 * @var_idx: The trigger variable identifier
1539 *
1540 * Check the list of var_refs[] on the first hist trigger to see
1541 * whether any of them are references to the variable on the second
1542 * trigger.
1543 *
1544 * Return: The VAR_REF field referencing the variable if so, NULL if not
1545 */
1546static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1547                                       struct hist_trigger_data *var_data,
1548                                       unsigned int var_idx)
1549{
1550        struct hist_field *hist_field;
1551        unsigned int i;
1552
1553        for (i = 0; i < hist_data->n_var_refs; i++) {
1554                hist_field = hist_data->var_refs[i];
1555                if (check_field_for_var_ref(hist_field, var_data, var_idx))
1556                        return hist_field;
1557        }
1558
1559        return NULL;
1560}
1561
1562/**
1563 * find_any_var_ref - Check if there is a reference to a given trigger variable
1564 * @hist_data: The hist trigger
1565 * @var_idx: The trigger variable identifier
1566 *
1567 * Check to see whether the given variable is currently referenced by
1568 * any other trigger.
1569 *
1570 * The trigger the variable is defined on is explicitly excluded - the
1571 * assumption being that a self-reference doesn't prevent a trigger
1572 * from being removed.
1573 *
1574 * Return: The VAR_REF field referencing the variable if so, NULL if not
1575 */
1576static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1577                                           unsigned int var_idx)
1578{
1579        struct trace_array *tr = hist_data->event_file->tr;
1580        struct hist_field *found = NULL;
1581        struct hist_var_data *var_data;
1582
1583        list_for_each_entry(var_data, &tr->hist_vars, list) {
1584                if (var_data->hist_data == hist_data)
1585                        continue;
1586                found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1587                if (found)
1588                        break;
1589        }
1590
1591        return found;
1592}
1593
1594/**
1595 * check_var_refs - Check if there is a reference to any of trigger's variables
1596 * @hist_data: The hist trigger
1597 *
1598 * A trigger can define one or more variables.  If any one of them is
1599 * currently referenced by any other trigger, this function will
1600 * determine that.
1601
1602 * Typically used to determine whether or not a trigger can be removed
1603 * - if there are any references to a trigger's variables, it cannot.
1604 *
1605 * Return: True if there is a reference to any of trigger's variables
1606 */
1607static bool check_var_refs(struct hist_trigger_data *hist_data)
1608{
1609        struct hist_field *field;
1610        bool found = false;
1611        int i;
1612
1613        for_each_hist_field(i, hist_data) {
1614                field = hist_data->fields[i];
1615                if (field && field->flags & HIST_FIELD_FL_VAR) {
1616                        if (find_any_var_ref(hist_data, field->var.idx)) {
1617                                found = true;
1618                                break;
1619                        }
1620                }
1621        }
1622
1623        return found;
1624}
1625
1626static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1627{
1628        struct trace_array *tr = hist_data->event_file->tr;
1629        struct hist_var_data *var_data, *found = NULL;
1630
1631        list_for_each_entry(var_data, &tr->hist_vars, list) {
1632                if (var_data->hist_data == hist_data) {
1633                        found = var_data;
1634                        break;
1635                }
1636        }
1637
1638        return found;
1639}
1640
1641static bool field_has_hist_vars(struct hist_field *hist_field,
1642                                unsigned int level)
1643{
1644        int i;
1645
1646        if (level > 3)
1647                return false;
1648
1649        if (!hist_field)
1650                return false;
1651
1652        if (hist_field->flags & HIST_FIELD_FL_VAR ||
1653            hist_field->flags & HIST_FIELD_FL_VAR_REF)
1654                return true;
1655
1656        for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1657                struct hist_field *operand;
1658
1659                operand = hist_field->operands[i];
1660                if (field_has_hist_vars(operand, level + 1))
1661                        return true;
1662        }
1663
1664        return false;
1665}
1666
1667static bool has_hist_vars(struct hist_trigger_data *hist_data)
1668{
1669        struct hist_field *hist_field;
1670        int i;
1671
1672        for_each_hist_field(i, hist_data) {
1673                hist_field = hist_data->fields[i];
1674                if (field_has_hist_vars(hist_field, 0))
1675                        return true;
1676        }
1677
1678        return false;
1679}
1680
1681static int save_hist_vars(struct hist_trigger_data *hist_data)
1682{
1683        struct trace_array *tr = hist_data->event_file->tr;
1684        struct hist_var_data *var_data;
1685
1686        var_data = find_hist_vars(hist_data);
1687        if (var_data)
1688                return 0;
1689
1690        if (tracing_check_open_get_tr(tr))
1691                return -ENODEV;
1692
1693        var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1694        if (!var_data) {
1695                trace_array_put(tr);
1696                return -ENOMEM;
1697        }
1698
1699        var_data->hist_data = hist_data;
1700        list_add(&var_data->list, &tr->hist_vars);
1701
1702        return 0;
1703}
1704
1705static void remove_hist_vars(struct hist_trigger_data *hist_data)
1706{
1707        struct trace_array *tr = hist_data->event_file->tr;
1708        struct hist_var_data *var_data;
1709
1710        var_data = find_hist_vars(hist_data);
1711        if (!var_data)
1712                return;
1713
1714        if (WARN_ON(check_var_refs(hist_data)))
1715                return;
1716
1717        list_del(&var_data->list);
1718
1719        kfree(var_data);
1720
1721        trace_array_put(tr);
1722}
1723
1724static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1725                                         const char *var_name)
1726{
1727        struct hist_field *hist_field, *found = NULL;
1728        int i;
1729
1730        for_each_hist_field(i, hist_data) {
1731                hist_field = hist_data->fields[i];
1732                if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1733                    strcmp(hist_field->var.name, var_name) == 0) {
1734                        found = hist_field;
1735                        break;
1736                }
1737        }
1738
1739        return found;
1740}
1741
1742static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1743                                   struct trace_event_file *file,
1744                                   const char *var_name)
1745{
1746        struct hist_trigger_data *test_data;
1747        struct event_trigger_data *test;
1748        struct hist_field *hist_field;
1749
1750        hist_field = find_var_field(hist_data, var_name);
1751        if (hist_field)
1752                return hist_field;
1753
1754        list_for_each_entry_rcu(test, &file->triggers, list) {
1755                if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1756                        test_data = test->private_data;
1757                        hist_field = find_var_field(test_data, var_name);
1758                        if (hist_field)
1759                                return hist_field;
1760                }
1761        }
1762
1763        return NULL;
1764}
1765
1766static struct trace_event_file *find_var_file(struct trace_array *tr,
1767                                              char *system,
1768                                              char *event_name,
1769                                              char *var_name)
1770{
1771        struct hist_trigger_data *var_hist_data;
1772        struct hist_var_data *var_data;
1773        struct trace_event_file *file, *found = NULL;
1774
1775        if (system)
1776                return find_event_file(tr, system, event_name);
1777
1778        list_for_each_entry(var_data, &tr->hist_vars, list) {
1779                var_hist_data = var_data->hist_data;
1780                file = var_hist_data->event_file;
1781                if (file == found)
1782                        continue;
1783
1784                if (find_var_field(var_hist_data, var_name)) {
1785                        if (found) {
1786                                hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1787                                return NULL;
1788                        }
1789
1790                        found = file;
1791                }
1792        }
1793
1794        return found;
1795}
1796
1797static struct hist_field *find_file_var(struct trace_event_file *file,
1798                                        const char *var_name)
1799{
1800        struct hist_trigger_data *test_data;
1801        struct event_trigger_data *test;
1802        struct hist_field *hist_field;
1803
1804        list_for_each_entry_rcu(test, &file->triggers, list) {
1805                if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1806                        test_data = test->private_data;
1807                        hist_field = find_var_field(test_data, var_name);
1808                        if (hist_field)
1809                                return hist_field;
1810                }
1811        }
1812
1813        return NULL;
1814}
1815
1816static struct hist_field *
1817find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1818{
1819        struct trace_array *tr = hist_data->event_file->tr;
1820        struct hist_field *hist_field, *found = NULL;
1821        struct trace_event_file *file;
1822        unsigned int i;
1823
1824        for (i = 0; i < hist_data->n_actions; i++) {
1825                struct action_data *data = hist_data->actions[i];
1826
1827                if (data->handler == HANDLER_ONMATCH) {
1828                        char *system = data->match_data.event_system;
1829                        char *event_name = data->match_data.event;
1830
1831                        file = find_var_file(tr, system, event_name, var_name);
1832                        if (!file)
1833                                continue;
1834                        hist_field = find_file_var(file, var_name);
1835                        if (hist_field) {
1836                                if (found) {
1837                                        hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1838                                                 errpos(var_name));
1839                                        return ERR_PTR(-EINVAL);
1840                                }
1841
1842                                found = hist_field;
1843                        }
1844                }
1845        }
1846        return found;
1847}
1848
1849static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1850                                         char *system,
1851                                         char *event_name,
1852                                         char *var_name)
1853{
1854        struct trace_array *tr = hist_data->event_file->tr;
1855        struct hist_field *hist_field = NULL;
1856        struct trace_event_file *file;
1857
1858        if (!system || !event_name) {
1859                hist_field = find_match_var(hist_data, var_name);
1860                if (IS_ERR(hist_field))
1861                        return NULL;
1862                if (hist_field)
1863                        return hist_field;
1864        }
1865
1866        file = find_var_file(tr, system, event_name, var_name);
1867        if (!file)
1868                return NULL;
1869
1870        hist_field = find_file_var(file, var_name);
1871
1872        return hist_field;
1873}
1874
1875static u64 hist_field_var_ref(struct hist_field *hist_field,
1876                              struct tracing_map_elt *elt,
1877                              struct ring_buffer_event *rbe,
1878                              void *event)
1879{
1880        struct hist_elt_data *elt_data;
1881        u64 var_val = 0;
1882
1883        if (WARN_ON_ONCE(!elt))
1884                return var_val;
1885
1886        elt_data = elt->private_data;
1887        var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1888
1889        return var_val;
1890}
1891
1892static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1893                             u64 *var_ref_vals, bool self)
1894{
1895        struct hist_trigger_data *var_data;
1896        struct tracing_map_elt *var_elt;
1897        struct hist_field *hist_field;
1898        unsigned int i, var_idx;
1899        bool resolved = true;
1900        u64 var_val = 0;
1901
1902        for (i = 0; i < hist_data->n_var_refs; i++) {
1903                hist_field = hist_data->var_refs[i];
1904                var_idx = hist_field->var.idx;
1905                var_data = hist_field->var.hist_data;
1906
1907                if (var_data == NULL) {
1908                        resolved = false;
1909                        break;
1910                }
1911
1912                if ((self && var_data != hist_data) ||
1913                    (!self && var_data == hist_data))
1914                        continue;
1915
1916                var_elt = tracing_map_lookup(var_data->map, key);
1917                if (!var_elt) {
1918                        resolved = false;
1919                        break;
1920                }
1921
1922                if (!tracing_map_var_set(var_elt, var_idx)) {
1923                        resolved = false;
1924                        break;
1925                }
1926
1927                if (self || !hist_field->read_once)
1928                        var_val = tracing_map_read_var(var_elt, var_idx);
1929                else
1930                        var_val = tracing_map_read_var_once(var_elt, var_idx);
1931
1932                var_ref_vals[i] = var_val;
1933        }
1934
1935        return resolved;
1936}
1937
1938static const char *hist_field_name(struct hist_field *field,
1939                                   unsigned int level)
1940{
1941        const char *field_name = "";
1942
1943        if (level > 1)
1944                return field_name;
1945
1946        if (field->field)
1947                field_name = field->field->name;
1948        else if (field->flags & HIST_FIELD_FL_LOG2 ||
1949                 field->flags & HIST_FIELD_FL_ALIAS)
1950                field_name = hist_field_name(field->operands[0], ++level);
1951        else if (field->flags & HIST_FIELD_FL_CPU)
1952                field_name = "cpu";
1953        else if (field->flags & HIST_FIELD_FL_EXPR ||
1954                 field->flags & HIST_FIELD_FL_VAR_REF) {
1955                if (field->system) {
1956                        static char full_name[MAX_FILTER_STR_VAL];
1957
1958                        strcat(full_name, field->system);
1959                        strcat(full_name, ".");
1960                        strcat(full_name, field->event_name);
1961                        strcat(full_name, ".");
1962                        strcat(full_name, field->name);
1963                        field_name = full_name;
1964                } else
1965                        field_name = field->name;
1966        } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1967                field_name = "common_timestamp";
1968
1969        if (field_name == NULL)
1970                field_name = "";
1971
1972        return field_name;
1973}
1974
1975static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1976{
1977        hist_field_fn_t fn = NULL;
1978
1979        switch (field_size) {
1980        case 8:
1981                if (field_is_signed)
1982                        fn = hist_field_s64;
1983                else
1984                        fn = hist_field_u64;
1985                break;
1986        case 4:
1987                if (field_is_signed)
1988                        fn = hist_field_s32;
1989                else
1990                        fn = hist_field_u32;
1991                break;
1992        case 2:
1993                if (field_is_signed)
1994                        fn = hist_field_s16;
1995                else
1996                        fn = hist_field_u16;
1997                break;
1998        case 1:
1999                if (field_is_signed)
2000                        fn = hist_field_s8;
2001                else
2002                        fn = hist_field_u8;
2003                break;
2004        }
2005
2006        return fn;
2007}
2008
2009static int parse_map_size(char *str)
2010{
2011        unsigned long size, map_bits;
2012        int ret;
2013
2014        strsep(&str, "=");
2015        if (!str) {
2016                ret = -EINVAL;
2017                goto out;
2018        }
2019
2020        ret = kstrtoul(str, 0, &size);
2021        if (ret)
2022                goto out;
2023
2024        map_bits = ilog2(roundup_pow_of_two(size));
2025        if (map_bits < TRACING_MAP_BITS_MIN ||
2026            map_bits > TRACING_MAP_BITS_MAX)
2027                ret = -EINVAL;
2028        else
2029                ret = map_bits;
2030 out:
2031        return ret;
2032}
2033
2034static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2035{
2036        unsigned int i;
2037
2038        if (!attrs)
2039                return;
2040
2041        for (i = 0; i < attrs->n_assignments; i++)
2042                kfree(attrs->assignment_str[i]);
2043
2044        for (i = 0; i < attrs->n_actions; i++)
2045                kfree(attrs->action_str[i]);
2046
2047        kfree(attrs->name);
2048        kfree(attrs->sort_key_str);
2049        kfree(attrs->keys_str);
2050        kfree(attrs->vals_str);
2051        kfree(attrs->clock);
2052        kfree(attrs);
2053}
2054
2055static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2056{
2057        int ret = -EINVAL;
2058
2059        if (attrs->n_actions >= HIST_ACTIONS_MAX)
2060                return ret;
2061
2062        if ((str_has_prefix(str, "onmatch(")) ||
2063            (str_has_prefix(str, "onmax(")) ||
2064            (str_has_prefix(str, "onchange("))) {
2065                attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2066                if (!attrs->action_str[attrs->n_actions]) {
2067                        ret = -ENOMEM;
2068                        return ret;
2069                }
2070                attrs->n_actions++;
2071                ret = 0;
2072        }
2073        return ret;
2074}
2075
2076static int parse_assignment(struct trace_array *tr,
2077                            char *str, struct hist_trigger_attrs *attrs)
2078{
2079        int ret = 0;
2080
2081        if ((str_has_prefix(str, "key=")) ||
2082            (str_has_prefix(str, "keys="))) {
2083                attrs->keys_str = kstrdup(str, GFP_KERNEL);
2084                if (!attrs->keys_str) {
2085                        ret = -ENOMEM;
2086                        goto out;
2087                }
2088        } else if ((str_has_prefix(str, "val=")) ||
2089                   (str_has_prefix(str, "vals=")) ||
2090                   (str_has_prefix(str, "values="))) {
2091                attrs->vals_str = kstrdup(str, GFP_KERNEL);
2092                if (!attrs->vals_str) {
2093                        ret = -ENOMEM;
2094                        goto out;
2095                }
2096        } else if (str_has_prefix(str, "sort=")) {
2097                attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
2098                if (!attrs->sort_key_str) {
2099                        ret = -ENOMEM;
2100                        goto out;
2101                }
2102        } else if (str_has_prefix(str, "name=")) {
2103                attrs->name = kstrdup(str, GFP_KERNEL);
2104                if (!attrs->name) {
2105                        ret = -ENOMEM;
2106                        goto out;
2107                }
2108        } else if (str_has_prefix(str, "clock=")) {
2109                strsep(&str, "=");
2110                if (!str) {
2111                        ret = -EINVAL;
2112                        goto out;
2113                }
2114
2115                str = strstrip(str);
2116                attrs->clock = kstrdup(str, GFP_KERNEL);
2117                if (!attrs->clock) {
2118                        ret = -ENOMEM;
2119                        goto out;
2120                }
2121        } else if (str_has_prefix(str, "size=")) {
2122                int map_bits = parse_map_size(str);
2123
2124                if (map_bits < 0) {
2125                        ret = map_bits;
2126                        goto out;
2127                }
2128                attrs->map_bits = map_bits;
2129        } else {
2130                char *assignment;
2131
2132                if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2133                        hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
2134                        ret = -EINVAL;
2135                        goto out;
2136                }
2137
2138                assignment = kstrdup(str, GFP_KERNEL);
2139                if (!assignment) {
2140                        ret = -ENOMEM;
2141                        goto out;
2142                }
2143
2144                attrs->assignment_str[attrs->n_assignments++] = assignment;
2145        }
2146 out:
2147        return ret;
2148}
2149
2150static struct hist_trigger_attrs *
2151parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
2152{
2153        struct hist_trigger_attrs *attrs;
2154        int ret = 0;
2155
2156        attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2157        if (!attrs)
2158                return ERR_PTR(-ENOMEM);
2159
2160        while (trigger_str) {
2161                char *str = strsep(&trigger_str, ":");
2162
2163                if (strchr(str, '=')) {
2164                        ret = parse_assignment(tr, str, attrs);
2165                        if (ret)
2166                                goto free;
2167                } else if (strcmp(str, "pause") == 0)
2168                        attrs->pause = true;
2169                else if ((strcmp(str, "cont") == 0) ||
2170                         (strcmp(str, "continue") == 0))
2171                        attrs->cont = true;
2172                else if (strcmp(str, "clear") == 0)
2173                        attrs->clear = true;
2174                else {
2175                        ret = parse_action(str, attrs);
2176                        if (ret)
2177                                goto free;
2178                }
2179        }
2180
2181        if (!attrs->keys_str) {
2182                ret = -EINVAL;
2183                goto free;
2184        }
2185
2186        if (!attrs->clock) {
2187                attrs->clock = kstrdup("global", GFP_KERNEL);
2188                if (!attrs->clock) {
2189                        ret = -ENOMEM;
2190                        goto free;
2191                }
2192        }
2193
2194        return attrs;
2195 free:
2196        destroy_hist_trigger_attrs(attrs);
2197
2198        return ERR_PTR(ret);
2199}
2200
2201static inline void save_comm(char *comm, struct task_struct *task)
2202{
2203        if (!task->pid) {
2204                strcpy(comm, "<idle>");
2205                return;
2206        }
2207
2208        if (WARN_ON_ONCE(task->pid < 0)) {
2209                strcpy(comm, "<XXX>");
2210                return;
2211        }
2212
2213        strncpy(comm, task->comm, TASK_COMM_LEN);
2214}
2215
2216static void hist_elt_data_free(struct hist_elt_data *elt_data)
2217{
2218        unsigned int i;
2219
2220        for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2221                kfree(elt_data->field_var_str[i]);
2222
2223        kfree(elt_data->comm);
2224        kfree(elt_data);
2225}
2226
2227static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2228{
2229        struct hist_elt_data *elt_data = elt->private_data;
2230
2231        hist_elt_data_free(elt_data);
2232}
2233
2234static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2235{
2236        struct hist_trigger_data *hist_data = elt->map->private_data;
2237        unsigned int size = TASK_COMM_LEN;
2238        struct hist_elt_data *elt_data;
2239        struct hist_field *key_field;
2240        unsigned int i, n_str;
2241
2242        elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2243        if (!elt_data)
2244                return -ENOMEM;
2245
2246        for_each_hist_key_field(i, hist_data) {
2247                key_field = hist_data->fields[i];
2248
2249                if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2250                        elt_data->comm = kzalloc(size, GFP_KERNEL);
2251                        if (!elt_data->comm) {
2252                                kfree(elt_data);
2253                                return -ENOMEM;
2254                        }
2255                        break;
2256                }
2257        }
2258
2259        n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2260
2261        size = STR_VAR_LEN_MAX;
2262
2263        for (i = 0; i < n_str; i++) {
2264                elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2265                if (!elt_data->field_var_str[i]) {
2266                        hist_elt_data_free(elt_data);
2267                        return -ENOMEM;
2268                }
2269        }
2270
2271        elt->private_data = elt_data;
2272
2273        return 0;
2274}
2275
2276static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2277{
2278        struct hist_elt_data *elt_data = elt->private_data;
2279
2280        if (elt_data->comm)
2281                save_comm(elt_data->comm, current);
2282}
2283
2284static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2285        .elt_alloc      = hist_trigger_elt_data_alloc,
2286        .elt_free       = hist_trigger_elt_data_free,
2287        .elt_init       = hist_trigger_elt_data_init,
2288};
2289
2290static const char *get_hist_field_flags(struct hist_field *hist_field)
2291{
2292        const char *flags_str = NULL;
2293
2294        if (hist_field->flags & HIST_FIELD_FL_HEX)
2295                flags_str = "hex";
2296        else if (hist_field->flags & HIST_FIELD_FL_SYM)
2297                flags_str = "sym";
2298        else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2299                flags_str = "sym-offset";
2300        else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2301                flags_str = "execname";
2302        else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2303                flags_str = "syscall";
2304        else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2305                flags_str = "log2";
2306        else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2307                flags_str = "usecs";
2308
2309        return flags_str;
2310}
2311
2312static void expr_field_str(struct hist_field *field, char *expr)
2313{
2314        if (field->flags & HIST_FIELD_FL_VAR_REF)
2315                strcat(expr, "$");
2316
2317        strcat(expr, hist_field_name(field, 0));
2318
2319        if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2320                const char *flags_str = get_hist_field_flags(field);
2321
2322                if (flags_str) {
2323                        strcat(expr, ".");
2324                        strcat(expr, flags_str);
2325                }
2326        }
2327}
2328
2329static char *expr_str(struct hist_field *field, unsigned int level)
2330{
2331        char *expr;
2332
2333        if (level > 1)
2334                return NULL;
2335
2336        expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2337        if (!expr)
2338                return NULL;
2339
2340        if (!field->operands[0]) {
2341                expr_field_str(field, expr);
2342                return expr;
2343        }
2344
2345        if (field->operator == FIELD_OP_UNARY_MINUS) {
2346                char *subexpr;
2347
2348                strcat(expr, "-(");
2349                subexpr = expr_str(field->operands[0], ++level);
2350                if (!subexpr) {
2351                        kfree(expr);
2352                        return NULL;
2353                }
2354                strcat(expr, subexpr);
2355                strcat(expr, ")");
2356
2357                kfree(subexpr);
2358
2359                return expr;
2360        }
2361
2362        expr_field_str(field->operands[0], expr);
2363
2364        switch (field->operator) {
2365        case FIELD_OP_MINUS:
2366                strcat(expr, "-");
2367                break;
2368        case FIELD_OP_PLUS:
2369                strcat(expr, "+");
2370                break;
2371        default:
2372                kfree(expr);
2373                return NULL;
2374        }
2375
2376        expr_field_str(field->operands[1], expr);
2377
2378        return expr;
2379}
2380
2381static int contains_operator(char *str)
2382{
2383        enum field_op_id field_op = FIELD_OP_NONE;
2384        char *op;
2385
2386        op = strpbrk(str, "+-");
2387        if (!op)
2388                return FIELD_OP_NONE;
2389
2390        switch (*op) {
2391        case '-':
2392                if (*str == '-')
2393                        field_op = FIELD_OP_UNARY_MINUS;
2394                else
2395                        field_op = FIELD_OP_MINUS;
2396                break;
2397        case '+':
2398                field_op = FIELD_OP_PLUS;
2399                break;
2400        default:
2401                break;
2402        }
2403
2404        return field_op;
2405}
2406
2407static void __destroy_hist_field(struct hist_field *hist_field)
2408{
2409        kfree(hist_field->var.name);
2410        kfree(hist_field->name);
2411        kfree(hist_field->type);
2412
2413        kfree(hist_field);
2414}
2415
2416static void destroy_hist_field(struct hist_field *hist_field,
2417                               unsigned int level)
2418{
2419        unsigned int i;
2420
2421        if (level > 3)
2422                return;
2423
2424        if (!hist_field)
2425                return;
2426
2427        if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2428                return; /* var refs will be destroyed separately */
2429
2430        for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2431                destroy_hist_field(hist_field->operands[i], level + 1);
2432
2433        __destroy_hist_field(hist_field);
2434}
2435
2436static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2437                                            struct ftrace_event_field *field,
2438                                            unsigned long flags,
2439                                            char *var_name)
2440{
2441        struct hist_field *hist_field;
2442
2443        if (field && is_function_field(field))
2444                return NULL;
2445
2446        hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2447        if (!hist_field)
2448                return NULL;
2449
2450        hist_field->hist_data = hist_data;
2451
2452        if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2453                goto out; /* caller will populate */
2454
2455        if (flags & HIST_FIELD_FL_VAR_REF) {
2456                hist_field->fn = hist_field_var_ref;
2457                goto out;
2458        }
2459
2460        if (flags & HIST_FIELD_FL_HITCOUNT) {
2461                hist_field->fn = hist_field_counter;
2462                hist_field->size = sizeof(u64);
2463                hist_field->type = kstrdup("u64", GFP_KERNEL);
2464                if (!hist_field->type)
2465                        goto free;
2466                goto out;
2467        }
2468
2469        if (flags & HIST_FIELD_FL_STACKTRACE) {
2470                hist_field->fn = hist_field_none;
2471                goto out;
2472        }
2473
2474        if (flags & HIST_FIELD_FL_LOG2) {
2475                unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2476                hist_field->fn = hist_field_log2;
2477                hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2478                hist_field->size = hist_field->operands[0]->size;
2479                hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2480                if (!hist_field->type)
2481                        goto free;
2482                goto out;
2483        }
2484
2485        if (flags & HIST_FIELD_FL_TIMESTAMP) {
2486                hist_field->fn = hist_field_timestamp;
2487                hist_field->size = sizeof(u64);
2488                hist_field->type = kstrdup("u64", GFP_KERNEL);
2489                if (!hist_field->type)
2490                        goto free;
2491                goto out;
2492        }
2493
2494        if (flags & HIST_FIELD_FL_CPU) {
2495                hist_field->fn = hist_field_cpu;
2496                hist_field->size = sizeof(int);
2497                hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2498                if (!hist_field->type)
2499                        goto free;
2500                goto out;
2501        }
2502
2503        if (WARN_ON_ONCE(!field))
2504                goto out;
2505
2506        if (is_string_field(field)) {
2507                flags |= HIST_FIELD_FL_STRING;
2508
2509                hist_field->size = MAX_FILTER_STR_VAL;
2510                hist_field->type = kstrdup(field->type, GFP_KERNEL);
2511                if (!hist_field->type)
2512                        goto free;
2513
2514                if (field->filter_type == FILTER_STATIC_STRING)
2515                        hist_field->fn = hist_field_string;
2516                else if (field->filter_type == FILTER_DYN_STRING)
2517                        hist_field->fn = hist_field_dynstring;
2518                else
2519                        hist_field->fn = hist_field_pstring;
2520        } else {
2521                hist_field->size = field->size;
2522                hist_field->is_signed = field->is_signed;
2523                hist_field->type = kstrdup(field->type, GFP_KERNEL);
2524                if (!hist_field->type)
2525                        goto free;
2526
2527                hist_field->fn = select_value_fn(field->size,
2528                                                 field->is_signed);
2529                if (!hist_field->fn) {
2530                        destroy_hist_field(hist_field, 0);
2531                        return NULL;
2532                }
2533        }
2534 out:
2535        hist_field->field = field;
2536        hist_field->flags = flags;
2537
2538        if (var_name) {
2539                hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2540                if (!hist_field->var.name)
2541                        goto free;
2542        }
2543
2544        return hist_field;
2545 free:
2546        destroy_hist_field(hist_field, 0);
2547        return NULL;
2548}
2549
2550static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2551{
2552        unsigned int i;
2553
2554        for (i = 0; i < HIST_FIELDS_MAX; i++) {
2555                if (hist_data->fields[i]) {
2556                        destroy_hist_field(hist_data->fields[i], 0);
2557                        hist_data->fields[i] = NULL;
2558                }
2559        }
2560
2561        for (i = 0; i < hist_data->n_var_refs; i++) {
2562                WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2563                __destroy_hist_field(hist_data->var_refs[i]);
2564                hist_data->var_refs[i] = NULL;
2565        }
2566}
2567
2568static int init_var_ref(struct hist_field *ref_field,
2569                        struct hist_field *var_field,
2570                        char *system, char *event_name)
2571{
2572        int err = 0;
2573
2574        ref_field->var.idx = var_field->var.idx;
2575        ref_field->var.hist_data = var_field->hist_data;
2576        ref_field->size = var_field->size;
2577        ref_field->is_signed = var_field->is_signed;
2578        ref_field->flags |= var_field->flags &
2579                (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2580
2581        if (system) {
2582                ref_field->system = kstrdup(system, GFP_KERNEL);
2583                if (!ref_field->system)
2584                        return -ENOMEM;
2585        }
2586
2587        if (event_name) {
2588                ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2589                if (!ref_field->event_name) {
2590                        err = -ENOMEM;
2591                        goto free;
2592                }
2593        }
2594
2595        if (var_field->var.name) {
2596                ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2597                if (!ref_field->name) {
2598                        err = -ENOMEM;
2599                        goto free;
2600                }
2601        } else if (var_field->name) {
2602                ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2603                if (!ref_field->name) {
2604                        err = -ENOMEM;
2605                        goto free;
2606                }
2607        }
2608
2609        ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2610        if (!ref_field->type) {
2611                err = -ENOMEM;
2612                goto free;
2613        }
2614 out:
2615        return err;
2616 free:
2617        kfree(ref_field->system);
2618        kfree(ref_field->event_name);
2619        kfree(ref_field->name);
2620
2621        goto out;
2622}
2623
2624/**
2625 * create_var_ref - Create a variable reference and attach it to trigger
2626 * @hist_data: The trigger that will be referencing the variable
2627 * @var_field: The VAR field to create a reference to
2628 * @system: The optional system string
2629 * @event_name: The optional event_name string
2630 *
2631 * Given a variable hist_field, create a VAR_REF hist_field that
2632 * represents a reference to it.
2633 *
2634 * This function also adds the reference to the trigger that
2635 * now references the variable.
2636 *
2637 * Return: The VAR_REF field if successful, NULL if not
2638 */
2639static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2640                                         struct hist_field *var_field,
2641                                         char *system, char *event_name)
2642{
2643        unsigned long flags = HIST_FIELD_FL_VAR_REF;
2644        struct hist_field *ref_field;
2645
2646        ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2647        if (ref_field) {
2648                if (init_var_ref(ref_field, var_field, system, event_name)) {
2649                        destroy_hist_field(ref_field, 0);
2650                        return NULL;
2651                }
2652
2653                hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2654                ref_field->var_ref_idx = hist_data->n_var_refs++;
2655        }
2656
2657        return ref_field;
2658}
2659
2660static bool is_var_ref(char *var_name)
2661{
2662        if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2663                return false;
2664
2665        return true;
2666}
2667
2668static char *field_name_from_var(struct hist_trigger_data *hist_data,
2669                                 char *var_name)
2670{
2671        char *name, *field;
2672        unsigned int i;
2673
2674        for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2675                name = hist_data->attrs->var_defs.name[i];
2676
2677                if (strcmp(var_name, name) == 0) {
2678                        field = hist_data->attrs->var_defs.expr[i];
2679                        if (contains_operator(field) || is_var_ref(field))
2680                                continue;
2681                        return field;
2682                }
2683        }
2684
2685        return NULL;
2686}
2687
2688static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2689                                 char *system, char *event_name,
2690                                 char *var_name)
2691{
2692        struct trace_event_call *call;
2693
2694        if (system && event_name) {
2695                call = hist_data->event_file->event_call;
2696
2697                if (strcmp(system, call->class->system) != 0)
2698                        return NULL;
2699
2700                if (strcmp(event_name, trace_event_name(call)) != 0)
2701                        return NULL;
2702        }
2703
2704        if (!!system != !!event_name)
2705                return NULL;
2706
2707        if (!is_var_ref(var_name))
2708                return NULL;
2709
2710        var_name++;
2711
2712        return field_name_from_var(hist_data, var_name);
2713}
2714
2715static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2716                                        char *system, char *event_name,
2717                                        char *var_name)
2718{
2719        struct hist_field *var_field = NULL, *ref_field = NULL;
2720        struct trace_array *tr = hist_data->event_file->tr;
2721
2722        if (!is_var_ref(var_name))
2723                return NULL;
2724
2725        var_name++;
2726
2727        var_field = find_event_var(hist_data, system, event_name, var_name);
2728        if (var_field)
2729                ref_field = create_var_ref(hist_data, var_field,
2730                                           system, event_name);
2731
2732        if (!ref_field)
2733                hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2734
2735        return ref_field;
2736}
2737
2738static struct ftrace_event_field *
2739parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2740            char *field_str, unsigned long *flags)
2741{
2742        struct ftrace_event_field *field = NULL;
2743        char *field_name, *modifier, *str;
2744        struct trace_array *tr = file->tr;
2745
2746        modifier = str = kstrdup(field_str, GFP_KERNEL);
2747        if (!modifier)
2748                return ERR_PTR(-ENOMEM);
2749
2750        field_name = strsep(&modifier, ".");
2751        if (modifier) {
2752                if (strcmp(modifier, "hex") == 0)
2753                        *flags |= HIST_FIELD_FL_HEX;
2754                else if (strcmp(modifier, "sym") == 0)
2755                        *flags |= HIST_FIELD_FL_SYM;
2756                else if (strcmp(modifier, "sym-offset") == 0)
2757                        *flags |= HIST_FIELD_FL_SYM_OFFSET;
2758                else if ((strcmp(modifier, "execname") == 0) &&
2759                         (strcmp(field_name, "common_pid") == 0))
2760                        *flags |= HIST_FIELD_FL_EXECNAME;
2761                else if (strcmp(modifier, "syscall") == 0)
2762                        *flags |= HIST_FIELD_FL_SYSCALL;
2763                else if (strcmp(modifier, "log2") == 0)
2764                        *flags |= HIST_FIELD_FL_LOG2;
2765                else if (strcmp(modifier, "usecs") == 0)
2766                        *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2767                else {
2768                        hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2769                        field = ERR_PTR(-EINVAL);
2770                        goto out;
2771                }
2772        }
2773
2774        if (strcmp(field_name, "common_timestamp") == 0) {
2775                *flags |= HIST_FIELD_FL_TIMESTAMP;
2776                hist_data->enable_timestamps = true;
2777                if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2778                        hist_data->attrs->ts_in_usecs = true;
2779        } else if (strcmp(field_name, "cpu") == 0)
2780                *flags |= HIST_FIELD_FL_CPU;
2781        else {
2782                field = trace_find_event_field(file->event_call, field_name);
2783                if (!field || !field->size) {
2784                        hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
2785                        field = ERR_PTR(-EINVAL);
2786                        goto out;
2787                }
2788        }
2789 out:
2790        kfree(str);
2791
2792        return field;
2793}
2794
2795static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2796                                       struct hist_field *var_ref,
2797                                       char *var_name)
2798{
2799        struct hist_field *alias = NULL;
2800        unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2801
2802        alias = create_hist_field(hist_data, NULL, flags, var_name);
2803        if (!alias)
2804                return NULL;
2805
2806        alias->fn = var_ref->fn;
2807        alias->operands[0] = var_ref;
2808
2809        if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2810                destroy_hist_field(alias, 0);
2811                return NULL;
2812        }
2813
2814        alias->var_ref_idx = var_ref->var_ref_idx;
2815
2816        return alias;
2817}
2818
2819static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2820                                     struct trace_event_file *file, char *str,
2821                                     unsigned long *flags, char *var_name)
2822{
2823        char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2824        struct ftrace_event_field *field = NULL;
2825        struct hist_field *hist_field = NULL;
2826        int ret = 0;
2827
2828        s = strchr(str, '.');
2829        if (s) {
2830                s = strchr(++s, '.');
2831                if (s) {
2832                        ref_system = strsep(&str, ".");
2833                        if (!str) {
2834                                ret = -EINVAL;
2835                                goto out;
2836                        }
2837                        ref_event = strsep(&str, ".");
2838                        if (!str) {
2839                                ret = -EINVAL;
2840                                goto out;
2841                        }
2842                        ref_var = str;
2843                }
2844        }
2845
2846        s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2847        if (!s) {
2848                hist_field = parse_var_ref(hist_data, ref_system,
2849                                           ref_event, ref_var);
2850                if (hist_field) {
2851                        if (var_name) {
2852                                hist_field = create_alias(hist_data, hist_field, var_name);
2853                                if (!hist_field) {
2854                                        ret = -ENOMEM;
2855                                        goto out;
2856                                }
2857                        }
2858                        return hist_field;
2859                }
2860        } else
2861                str = s;
2862
2863        field = parse_field(hist_data, file, str, flags);
2864        if (IS_ERR(field)) {
2865                ret = PTR_ERR(field);
2866                goto out;
2867        }
2868
2869        hist_field = create_hist_field(hist_data, field, *flags, var_name);
2870        if (!hist_field) {
2871                ret = -ENOMEM;
2872                goto out;
2873        }
2874
2875        return hist_field;
2876 out:
2877        return ERR_PTR(ret);
2878}
2879
2880static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2881                                     struct trace_event_file *file,
2882                                     char *str, unsigned long flags,
2883                                     char *var_name, unsigned int level);
2884
2885static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2886                                      struct trace_event_file *file,
2887                                      char *str, unsigned long flags,
2888                                      char *var_name, unsigned int level)
2889{
2890        struct hist_field *operand1, *expr = NULL;
2891        unsigned long operand_flags;
2892        int ret = 0;
2893        char *s;
2894
2895        /* we support only -(xxx) i.e. explicit parens required */
2896
2897        if (level > 3) {
2898                hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2899                ret = -EINVAL;
2900                goto free;
2901        }
2902
2903        str++; /* skip leading '-' */
2904
2905        s = strchr(str, '(');
2906        if (s)
2907                str++;
2908        else {
2909                ret = -EINVAL;
2910                goto free;
2911        }
2912
2913        s = strrchr(str, ')');
2914        if (s)
2915                *s = '\0';
2916        else {
2917                ret = -EINVAL; /* no closing ')' */
2918                goto free;
2919        }
2920
2921        flags |= HIST_FIELD_FL_EXPR;
2922        expr = create_hist_field(hist_data, NULL, flags, var_name);
2923        if (!expr) {
2924                ret = -ENOMEM;
2925                goto free;
2926        }
2927
2928        operand_flags = 0;
2929        operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2930        if (IS_ERR(operand1)) {
2931                ret = PTR_ERR(operand1);
2932                goto free;
2933        }
2934
2935        expr->flags |= operand1->flags &
2936                (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2937        expr->fn = hist_field_unary_minus;
2938        expr->operands[0] = operand1;
2939        expr->operator = FIELD_OP_UNARY_MINUS;
2940        expr->name = expr_str(expr, 0);
2941        expr->type = kstrdup(operand1->type, GFP_KERNEL);
2942        if (!expr->type) {
2943                ret = -ENOMEM;
2944                goto free;
2945        }
2946
2947        return expr;
2948 free:
2949        destroy_hist_field(expr, 0);
2950        return ERR_PTR(ret);
2951}
2952
2953static int check_expr_operands(struct trace_array *tr,
2954                               struct hist_field *operand1,
2955                               struct hist_field *operand2)
2956{
2957        unsigned long operand1_flags = operand1->flags;
2958        unsigned long operand2_flags = operand2->flags;
2959
2960        if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2961            (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2962                struct hist_field *var;
2963
2964                var = find_var_field(operand1->var.hist_data, operand1->name);
2965                if (!var)
2966                        return -EINVAL;
2967                operand1_flags = var->flags;
2968        }
2969
2970        if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2971            (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2972                struct hist_field *var;
2973
2974                var = find_var_field(operand2->var.hist_data, operand2->name);
2975                if (!var)
2976                        return -EINVAL;
2977                operand2_flags = var->flags;
2978        }
2979
2980        if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2981            (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2982                hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2983                return -EINVAL;
2984        }
2985
2986        return 0;
2987}
2988
2989static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2990                                     struct trace_event_file *file,
2991                                     char *str, unsigned long flags,
2992                                     char *var_name, unsigned int level)
2993{
2994        struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2995        unsigned long operand_flags;
2996        int field_op, ret = -EINVAL;
2997        char *sep, *operand1_str;
2998
2999        if (level > 3) {
3000                hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
3001                return ERR_PTR(-EINVAL);
3002        }
3003
3004        field_op = contains_operator(str);
3005
3006        if (field_op == FIELD_OP_NONE)
3007                return parse_atom(hist_data, file, str, &flags, var_name);
3008
3009        if (field_op == FIELD_OP_UNARY_MINUS)
3010                return parse_unary(hist_data, file, str, flags, var_name, ++level);
3011
3012        switch (field_op) {
3013        case FIELD_OP_MINUS:
3014                sep = "-";
3015                break;
3016        case FIELD_OP_PLUS:
3017                sep = "+";
3018                break;
3019        default:
3020                goto free;
3021        }
3022
3023        operand1_str = strsep(&str, sep);
3024        if (!operand1_str || !str)
3025                goto free;
3026
3027        operand_flags = 0;
3028        operand1 = parse_atom(hist_data, file, operand1_str,
3029                              &operand_flags, NULL);
3030        if (IS_ERR(operand1)) {
3031                ret = PTR_ERR(operand1);
3032                operand1 = NULL;
3033                goto free;
3034        }
3035
3036        /* rest of string could be another expression e.g. b+c in a+b+c */
3037        operand_flags = 0;
3038        operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3039        if (IS_ERR(operand2)) {
3040                ret = PTR_ERR(operand2);
3041                operand2 = NULL;
3042                goto free;
3043        }
3044
3045        ret = check_expr_operands(file->tr, operand1, operand2);
3046        if (ret)
3047                goto free;
3048
3049        flags |= HIST_FIELD_FL_EXPR;
3050
3051        flags |= operand1->flags &
3052                (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3053
3054        expr = create_hist_field(hist_data, NULL, flags, var_name);
3055        if (!expr) {
3056                ret = -ENOMEM;
3057                goto free;
3058        }
3059
3060        operand1->read_once = true;
3061        operand2->read_once = true;
3062
3063        expr->operands[0] = operand1;
3064        expr->operands[1] = operand2;
3065        expr->operator = field_op;
3066        expr->name = expr_str(expr, 0);
3067        expr->type = kstrdup(operand1->type, GFP_KERNEL);
3068        if (!expr->type) {
3069                ret = -ENOMEM;
3070                goto free;
3071        }
3072
3073        switch (field_op) {
3074        case FIELD_OP_MINUS:
3075                expr->fn = hist_field_minus;
3076                break;
3077        case FIELD_OP_PLUS:
3078                expr->fn = hist_field_plus;
3079                break;
3080        default:
3081                ret = -EINVAL;
3082                goto free;
3083        }
3084
3085        return expr;
3086 free:
3087        destroy_hist_field(operand1, 0);
3088        destroy_hist_field(operand2, 0);
3089        destroy_hist_field(expr, 0);
3090
3091        return ERR_PTR(ret);
3092}
3093
3094static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3095                                 struct trace_event_file *file)
3096{
3097        struct event_trigger_data *test;
3098
3099        list_for_each_entry_rcu(test, &file->triggers, list) {
3100                if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3101                        if (test->private_data == hist_data)
3102                                return test->filter_str;
3103                }
3104        }
3105
3106        return NULL;
3107}
3108
3109static struct event_command trigger_hist_cmd;
3110static int event_hist_trigger_func(struct event_command *cmd_ops,
3111                                   struct trace_event_file *file,
3112                                   char *glob, char *cmd, char *param);
3113
3114static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3115                            struct hist_trigger_data *hist_data,
3116                            unsigned int n_keys)
3117{
3118        struct hist_field *target_hist_field, *hist_field;
3119        unsigned int n, i, j;
3120
3121        if (hist_data->n_fields - hist_data->n_vals != n_keys)
3122                return false;
3123
3124        i = hist_data->n_vals;
3125        j = target_hist_data->n_vals;
3126
3127        for (n = 0; n < n_keys; n++) {
3128                hist_field = hist_data->fields[i + n];
3129                target_hist_field = target_hist_data->fields[j + n];
3130
3131                if (strcmp(hist_field->type, target_hist_field->type) != 0)
3132                        return false;
3133                if (hist_field->size != target_hist_field->size)
3134                        return false;
3135                if (hist_field->is_signed != target_hist_field->is_signed)
3136                        return false;
3137        }
3138
3139        return true;
3140}
3141
3142static struct hist_trigger_data *
3143find_compatible_hist(struct hist_trigger_data *target_hist_data,
3144                     struct trace_event_file *file)
3145{
3146        struct hist_trigger_data *hist_data;
3147        struct event_trigger_data *test;
3148        unsigned int n_keys;
3149
3150        n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3151
3152        list_for_each_entry_rcu(test, &file->triggers, list) {
3153                if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3154                        hist_data = test->private_data;
3155
3156                        if (compatible_keys(target_hist_data, hist_data, n_keys))
3157                                return hist_data;
3158                }
3159        }
3160
3161        return NULL;
3162}
3163
3164static struct trace_event_file *event_file(struct trace_array *tr,
3165                                           char *system, char *event_name)
3166{
3167        struct trace_event_file *file;
3168
3169        file = __find_event_file(tr, system, event_name);
3170        if (!file)
3171                return ERR_PTR(-EINVAL);
3172
3173        return file;
3174}
3175
3176static struct hist_field *
3177find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3178                         char *system, char *event_name, char *field_name)
3179{
3180        struct hist_field *event_var;
3181        char *synthetic_name;
3182
3183        synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3184        if (!synthetic_name)
3185                return ERR_PTR(-ENOMEM);
3186
3187        strcpy(synthetic_name, "synthetic_");
3188        strcat(synthetic_name, field_name);
3189
3190        event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3191
3192        kfree(synthetic_name);
3193
3194        return event_var;
3195}
3196
3197/**
3198 * create_field_var_hist - Automatically create a histogram and var for a field
3199 * @target_hist_data: The target hist trigger
3200 * @subsys_name: Optional subsystem name
3201 * @event_name: Optional event name
3202 * @field_name: The name of the field (and the resulting variable)
3203 *
3204 * Hist trigger actions fetch data from variables, not directly from
3205 * events.  However, for convenience, users are allowed to directly
3206 * specify an event field in an action, which will be automatically
3207 * converted into a variable on their behalf.
3208
3209 * If a user specifies a field on an event that isn't the event the
3210 * histogram currently being defined (the target event histogram), the
3211 * only way that can be accomplished is if a new hist trigger is
3212 * created and the field variable defined on that.
3213 *
3214 * This function creates a new histogram compatible with the target
3215 * event (meaning a histogram with the same key as the target
3216 * histogram), and creates a variable for the specified field, but
3217 * with 'synthetic_' prepended to the variable name in order to avoid
3218 * collision with normal field variables.
3219 *
3220 * Return: The variable created for the field.
3221 */
3222static struct hist_field *
3223create_field_var_hist(struct hist_trigger_data *target_hist_data,
3224                      char *subsys_name, char *event_name, char *field_name)
3225{
3226        struct trace_array *tr = target_hist_data->event_file->tr;
3227        struct hist_field *event_var = ERR_PTR(-EINVAL);
3228        struct hist_trigger_data *hist_data;
3229        unsigned int i, n, first = true;
3230        struct field_var_hist *var_hist;
3231        struct trace_event_file *file;
3232        struct hist_field *key_field;
3233        char *saved_filter;
3234        char *cmd;
3235        int ret;
3236
3237        if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3238                hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3239                return ERR_PTR(-EINVAL);
3240        }
3241
3242        file = event_file(tr, subsys_name, event_name);
3243
3244        if (IS_ERR(file)) {
3245                hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3246                ret = PTR_ERR(file);
3247                return ERR_PTR(ret);
3248        }
3249
3250        /*
3251         * Look for a histogram compatible with target.  We'll use the
3252         * found histogram specification to create a new matching
3253         * histogram with our variable on it.  target_hist_data is not
3254         * yet a registered histogram so we can't use that.
3255         */
3256        hist_data = find_compatible_hist(target_hist_data, file);
3257        if (!hist_data) {
3258                hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3259                return ERR_PTR(-EINVAL);
3260        }
3261
3262        /* See if a synthetic field variable has already been created */
3263        event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3264                                             event_name, field_name);
3265        if (!IS_ERR_OR_NULL(event_var))
3266                return event_var;
3267
3268        var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3269        if (!var_hist)
3270                return ERR_PTR(-ENOMEM);
3271
3272        cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3273        if (!cmd) {
3274                kfree(var_hist);
3275                return ERR_PTR(-ENOMEM);
3276        }
3277
3278        /* Use the same keys as the compatible histogram */
3279        strcat(cmd, "keys=");
3280
3281        for_each_hist_key_field(i, hist_data) {
3282                key_field = hist_data->fields[i];
3283                if (!first)
3284                        strcat(cmd, ",");
3285                strcat(cmd, key_field->field->name);
3286                first = false;
3287        }
3288
3289        /* Create the synthetic field variable specification */
3290        strcat(cmd, ":synthetic_");
3291        strcat(cmd, field_name);
3292        strcat(cmd, "=");
3293        strcat(cmd, field_name);
3294
3295        /* Use the same filter as the compatible histogram */
3296        saved_filter = find_trigger_filter(hist_data, file);
3297        if (saved_filter) {
3298                strcat(cmd, " if ");
3299                strcat(cmd, saved_filter);
3300        }
3301
3302        var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3303        if (!var_hist->cmd) {
3304                kfree(cmd);
3305                kfree(var_hist);
3306                return ERR_PTR(-ENOMEM);
3307        }
3308
3309        /* Save the compatible histogram information */
3310        var_hist->hist_data = hist_data;
3311
3312        /* Create the new histogram with our variable */
3313        ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3314                                      "", "hist", cmd);
3315        if (ret) {
3316                kfree(cmd);
3317                kfree(var_hist->cmd);
3318                kfree(var_hist);
3319                hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3320                return ERR_PTR(ret);
3321        }
3322
3323        kfree(cmd);
3324
3325        /* If we can't find the variable, something went wrong */
3326        event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3327                                             event_name, field_name);
3328        if (IS_ERR_OR_NULL(event_var)) {
3329                kfree(var_hist->cmd);
3330                kfree(var_hist);
3331                hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3332                return ERR_PTR(-EINVAL);
3333        }
3334
3335        n = target_hist_data->n_field_var_hists;
3336        target_hist_data->field_var_hists[n] = var_hist;
3337        target_hist_data->n_field_var_hists++;
3338
3339        return event_var;
3340}
3341
3342static struct hist_field *
3343find_target_event_var(struct hist_trigger_data *hist_data,
3344                      char *subsys_name, char *event_name, char *var_name)
3345{
3346        struct trace_event_file *file = hist_data->event_file;
3347        struct hist_field *hist_field = NULL;
3348
3349        if (subsys_name) {
3350                struct trace_event_call *call;
3351
3352                if (!event_name)
3353                        return NULL;
3354
3355                call = file->event_call;
3356
3357                if (strcmp(subsys_name, call->class->system) != 0)
3358                        return NULL;
3359
3360                if (strcmp(event_name, trace_event_name(call)) != 0)
3361                        return NULL;
3362        }
3363
3364        hist_field = find_var_field(hist_data, var_name);
3365
3366        return hist_field;
3367}
3368
3369static inline void __update_field_vars(struct tracing_map_elt *elt,
3370                                       struct ring_buffer_event *rbe,
3371                                       void *rec,
3372                                       struct field_var **field_vars,
3373                                       unsigned int n_field_vars,
3374                                       unsigned int field_var_str_start)
3375{
3376        struct hist_elt_data *elt_data = elt->private_data;
3377        unsigned int i, j, var_idx;
3378        u64 var_val;
3379
3380        for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3381                struct field_var *field_var = field_vars[i];
3382                struct hist_field *var = field_var->var;
3383                struct hist_field *val = field_var->val;
3384
3385                var_val = val->fn(val, elt, rbe, rec);
3386                var_idx = var->var.idx;
3387
3388                if (val->flags & HIST_FIELD_FL_STRING) {
3389                        char *str = elt_data->field_var_str[j++];
3390                        char *val_str = (char *)(uintptr_t)var_val;
3391
3392                        strscpy(str, val_str, STR_VAR_LEN_MAX);
3393                        var_val = (u64)(uintptr_t)str;
3394                }
3395                tracing_map_set_var(elt, var_idx, var_val);
3396        }
3397}
3398
3399static void update_field_vars(struct hist_trigger_data *hist_data,
3400                              struct tracing_map_elt *elt,
3401                              struct ring_buffer_event *rbe,
3402                              void *rec)
3403{
3404        __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3405                            hist_data->n_field_vars, 0);
3406}
3407
3408static void save_track_data_vars(struct hist_trigger_data *hist_data,
3409                                 struct tracing_map_elt *elt, void *rec,
3410                                 struct ring_buffer_event *rbe, void *key,
3411                                 struct action_data *data, u64 *var_ref_vals)
3412{
3413        __update_field_vars(elt, rbe, rec, hist_data->save_vars,
3414                            hist_data->n_save_vars, hist_data->n_field_var_str);
3415}
3416
3417static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3418                                     struct trace_event_file *file,
3419                                     char *name, int size, const char *type)
3420{
3421        struct hist_field *var;
3422        int idx;
3423
3424        if (find_var(hist_data, file, name) && !hist_data->remove) {
3425                var = ERR_PTR(-EINVAL);
3426                goto out;
3427        }
3428
3429        var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3430        if (!var) {
3431                var = ERR_PTR(-ENOMEM);
3432                goto out;
3433        }
3434
3435        idx = tracing_map_add_var(hist_data->map);
3436        if (idx < 0) {
3437                kfree(var);
3438                var = ERR_PTR(-EINVAL);
3439                goto out;
3440        }
3441
3442        var->flags = HIST_FIELD_FL_VAR;
3443        var->var.idx = idx;
3444        var->var.hist_data = var->hist_data = hist_data;
3445        var->size = size;
3446        var->var.name = kstrdup(name, GFP_KERNEL);
3447        var->type = kstrdup(type, GFP_KERNEL);
3448        if (!var->var.name || !var->type) {
3449                kfree(var->var.name);
3450                kfree(var->type);
3451                kfree(var);
3452                var = ERR_PTR(-ENOMEM);
3453        }
3454 out:
3455        return var;
3456}
3457
3458static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3459                                          struct trace_event_file *file,
3460                                          char *field_name)
3461{
3462        struct hist_field *val = NULL, *var = NULL;
3463        unsigned long flags = HIST_FIELD_FL_VAR;
3464        struct trace_array *tr = file->tr;
3465        struct field_var *field_var;
3466        int ret = 0;
3467
3468        if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3469                hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3470                ret = -EINVAL;
3471                goto err;
3472        }
3473
3474        val = parse_atom(hist_data, file, field_name, &flags, NULL);
3475        if (IS_ERR(val)) {
3476                hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3477                ret = PTR_ERR(val);
3478                goto err;
3479        }
3480
3481        var = create_var(hist_data, file, field_name, val->size, val->type);
3482        if (IS_ERR(var)) {
3483                hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3484                kfree(val);
3485                ret = PTR_ERR(var);
3486                goto err;
3487        }
3488
3489        field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3490        if (!field_var) {
3491                kfree(val);
3492                kfree(var);
3493                ret =  -ENOMEM;
3494                goto err;
3495        }
3496
3497        field_var->var = var;
3498        field_var->val = val;
3499 out:
3500        return field_var;
3501 err:
3502        field_var = ERR_PTR(ret);
3503        goto out;
3504}
3505
3506/**
3507 * create_target_field_var - Automatically create a variable for a field
3508 * @target_hist_data: The target hist trigger
3509 * @subsys_name: Optional subsystem name
3510 * @event_name: Optional event name
3511 * @var_name: The name of the field (and the resulting variable)
3512 *
3513 * Hist trigger actions fetch data from variables, not directly from
3514 * events.  However, for convenience, users are allowed to directly
3515 * specify an event field in an action, which will be automatically
3516 * converted into a variable on their behalf.
3517
3518 * This function creates a field variable with the name var_name on
3519 * the hist trigger currently being defined on the target event.  If
3520 * subsys_name and event_name are specified, this function simply
3521 * verifies that they do in fact match the target event subsystem and
3522 * event name.
3523 *
3524 * Return: The variable created for the field.
3525 */
3526static struct field_var *
3527create_target_field_var(struct hist_trigger_data *target_hist_data,
3528                        char *subsys_name, char *event_name, char *var_name)
3529{
3530        struct trace_event_file *file = target_hist_data->event_file;
3531
3532        if (subsys_name) {
3533                struct trace_event_call *call;
3534
3535                if (!event_name)
3536                        return NULL;
3537
3538                call = file->event_call;
3539
3540                if (strcmp(subsys_name, call->class->system) != 0)
3541                        return NULL;
3542
3543                if (strcmp(event_name, trace_event_name(call)) != 0)
3544                        return NULL;
3545        }
3546
3547        return create_field_var(target_hist_data, file, var_name);
3548}
3549
3550static bool check_track_val_max(u64 track_val, u64 var_val)
3551{
3552        if (var_val <= track_val)
3553                return false;
3554
3555        return true;
3556}
3557
3558static bool check_track_val_changed(u64 track_val, u64 var_val)
3559{
3560        if (var_val == track_val)
3561                return false;
3562
3563        return true;
3564}
3565
3566static u64 get_track_val(struct hist_trigger_data *hist_data,
3567                         struct tracing_map_elt *elt,
3568                         struct action_data *data)
3569{
3570        unsigned int track_var_idx = data->track_data.track_var->var.idx;
3571        u64 track_val;
3572
3573        track_val = tracing_map_read_var(elt, track_var_idx);
3574
3575        return track_val;
3576}
3577
3578static void save_track_val(struct hist_trigger_data *hist_data,
3579                           struct tracing_map_elt *elt,
3580                           struct action_data *data, u64 var_val)
3581{
3582        unsigned int track_var_idx = data->track_data.track_var->var.idx;
3583
3584        tracing_map_set_var(elt, track_var_idx, var_val);
3585}
3586
3587static void save_track_data(struct hist_trigger_data *hist_data,
3588                            struct tracing_map_elt *elt, void *rec,
3589                            struct ring_buffer_event *rbe, void *key,
3590                            struct action_data *data, u64 *var_ref_vals)
3591{
3592        if (data->track_data.save_data)
3593                data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3594}
3595
3596static bool check_track_val(struct tracing_map_elt *elt,
3597                            struct action_data *data,
3598                            u64 var_val)
3599{
3600        struct hist_trigger_data *hist_data;
3601        u64 track_val;
3602
3603        hist_data = data->track_data.track_var->hist_data;
3604        track_val = get_track_val(hist_data, elt, data);
3605
3606        return data->track_data.check_val(track_val, var_val);
3607}
3608
3609#ifdef CONFIG_TRACER_SNAPSHOT
3610static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3611{
3612        /* called with tr->max_lock held */
3613        struct track_data *track_data = tr->cond_snapshot->cond_data;
3614        struct hist_elt_data *elt_data, *track_elt_data;
3615        struct snapshot_context *context = cond_data;
3616        struct action_data *action;
3617        u64 track_val;
3618
3619        if (!track_data)
3620                return false;
3621
3622        action = track_data->action_data;
3623
3624        track_val = get_track_val(track_data->hist_data, context->elt,
3625                                  track_data->action_data);
3626
3627        if (!action->track_data.check_val(track_data->track_val, track_val))
3628                return false;
3629
3630        track_data->track_val = track_val;
3631        memcpy(track_data->key, context->key, track_data->key_len);
3632
3633        elt_data = context->elt->private_data;
3634        track_elt_data = track_data->elt.private_data;
3635        if (elt_data->comm)
3636                strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3637
3638        track_data->updated = true;
3639
3640        return true;
3641}
3642
3643static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3644                                     struct tracing_map_elt *elt, void *rec,
3645                                     struct ring_buffer_event *rbe, void *key,
3646                                     struct action_data *data,
3647                                     u64 *var_ref_vals)
3648{
3649        struct trace_event_file *file = hist_data->event_file;
3650        struct snapshot_context context;
3651
3652        context.elt = elt;
3653        context.key = key;
3654
3655        tracing_snapshot_cond(file->tr, &context);
3656}
3657
3658static void hist_trigger_print_key(struct seq_file *m,
3659                                   struct hist_trigger_data *hist_data,
3660                                   void *key,
3661                                   struct tracing_map_elt *elt);
3662
3663static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3664{
3665        unsigned int i;
3666
3667        if (!hist_data->n_actions)
3668                return NULL;
3669
3670        for (i = 0; i < hist_data->n_actions; i++) {
3671                struct action_data *data = hist_data->actions[i];
3672
3673                if (data->action == ACTION_SNAPSHOT)
3674                        return data;
3675        }
3676
3677        return NULL;
3678}
3679
3680static void track_data_snapshot_print(struct seq_file *m,
3681                                      struct hist_trigger_data *hist_data)
3682{
3683        struct trace_event_file *file = hist_data->event_file;
3684        struct track_data *track_data;
3685        struct action_data *action;
3686
3687        track_data = tracing_cond_snapshot_data(file->tr);
3688        if (!track_data)
3689                return;
3690
3691        if (!track_data->updated)
3692                return;
3693
3694        action = snapshot_action(hist_data);
3695        if (!action)
3696                return;
3697
3698        seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3699        seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3700                   action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3701                   action->track_data.var_str, track_data->track_val);
3702
3703        seq_puts(m, "\ttriggered by event with key: ");
3704        hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3705        seq_putc(m, '\n');
3706}
3707#else
3708static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3709{
3710        return false;
3711}
3712static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3713                                     struct tracing_map_elt *elt, void *rec,
3714                                     struct ring_buffer_event *rbe, void *key,
3715                                     struct action_data *data,
3716                                     u64 *var_ref_vals) {}
3717static void track_data_snapshot_print(struct seq_file *m,
3718                                      struct hist_trigger_data *hist_data) {}
3719#endif /* CONFIG_TRACER_SNAPSHOT */
3720
3721static void track_data_print(struct seq_file *m,
3722                             struct hist_trigger_data *hist_data,
3723                             struct tracing_map_elt *elt,
3724                             struct action_data *data)
3725{
3726        u64 track_val = get_track_val(hist_data, elt, data);
3727        unsigned int i, save_var_idx;
3728
3729        if (data->handler == HANDLER_ONMAX)
3730                seq_printf(m, "\n\tmax: %10llu", track_val);
3731        else if (data->handler == HANDLER_ONCHANGE)
3732                seq_printf(m, "\n\tchanged: %10llu", track_val);
3733
3734        if (data->action == ACTION_SNAPSHOT)
3735                return;
3736
3737        for (i = 0; i < hist_data->n_save_vars; i++) {
3738                struct hist_field *save_val = hist_data->save_vars[i]->val;
3739                struct hist_field *save_var = hist_data->save_vars[i]->var;
3740                u64 val;
3741
3742                save_var_idx = save_var->var.idx;
3743
3744                val = tracing_map_read_var(elt, save_var_idx);
3745
3746                if (save_val->flags & HIST_FIELD_FL_STRING) {
3747                        seq_printf(m, "  %s: %-32s", save_var->var.name,
3748                                   (char *)(uintptr_t)(val));
3749                } else
3750                        seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3751        }
3752}
3753
3754static void ontrack_action(struct hist_trigger_data *hist_data,
3755                           struct tracing_map_elt *elt, void *rec,
3756                           struct ring_buffer_event *rbe, void *key,
3757                           struct action_data *data, u64 *var_ref_vals)
3758{
3759        u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3760
3761        if (check_track_val(elt, data, var_val)) {
3762                save_track_val(hist_data, elt, data, var_val);
3763                save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3764        }
3765}
3766
3767static void action_data_destroy(struct action_data *data)
3768{
3769        unsigned int i;
3770
3771        lockdep_assert_held(&event_mutex);
3772
3773        kfree(data->action_name);
3774
3775        for (i = 0; i < data->n_params; i++)
3776                kfree(data->params[i]);
3777
3778        if (data->synth_event)
3779                data->synth_event->ref--;
3780
3781        kfree(data->synth_event_name);
3782
3783        kfree(data);
3784}
3785
3786static void track_data_destroy(struct hist_trigger_data *hist_data,
3787                               struct action_data *data)
3788{
3789        struct trace_event_file *file = hist_data->event_file;
3790
3791        destroy_hist_field(data->track_data.track_var, 0);
3792
3793        if (data->action == ACTION_SNAPSHOT) {
3794                struct track_data *track_data;
3795
3796                track_data = tracing_cond_snapshot_data(file->tr);
3797                if (track_data && track_data->hist_data == hist_data) {
3798                        tracing_snapshot_cond_disable(file->tr);
3799                        track_data_free(track_data);
3800                }
3801        }
3802
3803        kfree(data->track_data.var_str);
3804
3805        action_data_destroy(data);
3806}
3807
3808static int action_create(struct hist_trigger_data *hist_data,
3809                         struct action_data *data);
3810
3811static int track_data_create(struct hist_trigger_data *hist_data,
3812                             struct action_data *data)
3813{
3814        struct hist_field *var_field, *ref_field, *track_var = NULL;
3815        struct trace_event_file *file = hist_data->event_file;
3816        struct trace_array *tr = file->tr;
3817        char *track_data_var_str;
3818        int ret = 0;
3819
3820        track_data_var_str = data->track_data.var_str;
3821        if (track_data_var_str[0] != '$') {
3822                hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3823                return -EINVAL;
3824        }
3825        track_data_var_str++;
3826
3827        var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3828        if (!var_field) {
3829                hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3830                return -EINVAL;
3831        }
3832
3833        ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3834        if (!ref_field)
3835                return -ENOMEM;
3836
3837        data->track_data.var_ref = ref_field;
3838
3839        if (data->handler == HANDLER_ONMAX)
3840                track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3841        if (IS_ERR(track_var)) {
3842                hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3843                ret = PTR_ERR(track_var);
3844                goto out;
3845        }
3846
3847        if (data->handler == HANDLER_ONCHANGE)
3848                track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3849        if (IS_ERR(track_var)) {
3850                hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3851                ret = PTR_ERR(track_var);
3852                goto out;
3853        }
3854        data->track_data.track_var = track_var;
3855
3856        ret = action_create(hist_data, data);
3857 out:
3858        return ret;
3859}
3860
3861static int parse_action_params(struct trace_array *tr, char *params,
3862                               struct action_data *data)
3863{
3864        char *param, *saved_param;
3865        bool first_param = true;
3866        int ret = 0;
3867
3868        while (params) {
3869                if (data->n_params >= SYNTH_FIELDS_MAX) {
3870                        hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3871                        goto out;
3872                }
3873
3874                param = strsep(&params, ",");
3875                if (!param) {
3876                        hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3877                        ret = -EINVAL;
3878                        goto out;
3879                }
3880
3881                param = strstrip(param);
3882                if (strlen(param) < 2) {
3883                        hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3884                        ret = -EINVAL;
3885                        goto out;
3886                }
3887
3888                saved_param = kstrdup(param, GFP_KERNEL);
3889                if (!saved_param) {
3890                        ret = -ENOMEM;
3891                        goto out;
3892                }
3893
3894                if (first_param && data->use_trace_keyword) {
3895                        data->synth_event_name = saved_param;
3896                        first_param = false;
3897                        continue;
3898                }
3899                first_param = false;
3900
3901                data->params[data->n_params++] = saved_param;
3902        }
3903 out:
3904        return ret;
3905}
3906
3907static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3908                        enum handler_id handler)
3909{
3910        char *action_name;
3911        int ret = 0;
3912
3913        strsep(&str, ".");
3914        if (!str) {
3915                hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3916                ret = -EINVAL;
3917                goto out;
3918        }
3919
3920        action_name = strsep(&str, "(");
3921        if (!action_name || !str) {
3922                hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3923                ret = -EINVAL;
3924                goto out;
3925        }
3926
3927        if (str_has_prefix(action_name, "save")) {
3928                char *params = strsep(&str, ")");
3929
3930                if (!params) {
3931                        hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3932                        ret = -EINVAL;
3933                        goto out;
3934                }
3935
3936                ret = parse_action_params(tr, params, data);
3937                if (ret)
3938                        goto out;
3939
3940                if (handler == HANDLER_ONMAX)
3941                        data->track_data.check_val = check_track_val_max;
3942                else if (handler == HANDLER_ONCHANGE)
3943                        data->track_data.check_val = check_track_val_changed;
3944                else {
3945                        hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3946                        ret = -EINVAL;
3947                        goto out;
3948                }
3949
3950                data->track_data.save_data = save_track_data_vars;
3951                data->fn = ontrack_action;
3952                data->action = ACTION_SAVE;
3953        } else if (str_has_prefix(action_name, "snapshot")) {
3954                char *params = strsep(&str, ")");
3955
3956                if (!str) {
3957                        hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3958                        ret = -EINVAL;
3959                        goto out;
3960                }
3961
3962                if (handler == HANDLER_ONMAX)
3963                        data->track_data.check_val = check_track_val_max;
3964                else if (handler == HANDLER_ONCHANGE)
3965                        data->track_data.check_val = check_track_val_changed;
3966                else {
3967                        hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3968                        ret = -EINVAL;
3969                        goto out;
3970                }
3971
3972                data->track_data.save_data = save_track_data_snapshot;
3973                data->fn = ontrack_action;
3974                data->action = ACTION_SNAPSHOT;
3975        } else {
3976                char *params = strsep(&str, ")");
3977
3978                if (str_has_prefix(action_name, "trace"))
3979                        data->use_trace_keyword = true;
3980
3981                if (params) {
3982                        ret = parse_action_params(tr, params, data);
3983                        if (ret)
3984                                goto out;
3985                }
3986
3987                if (handler == HANDLER_ONMAX)
3988                        data->track_data.check_val = check_track_val_max;
3989                else if (handler == HANDLER_ONCHANGE)
3990                        data->track_data.check_val = check_track_val_changed;
3991
3992                if (handler != HANDLER_ONMATCH) {
3993                        data->track_data.save_data = action_trace;
3994                        data->fn = ontrack_action;
3995                } else
3996                        data->fn = action_trace;
3997
3998                data->action = ACTION_TRACE;
3999        }
4000
4001        data->action_name = kstrdup(action_name, GFP_KERNEL);
4002        if (!data->action_name) {
4003                ret = -ENOMEM;
4004                goto out;
4005        }
4006
4007        data->handler = handler;
4008 out:
4009        return ret;
4010}
4011
4012static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
4013                                            char *str, enum handler_id handler)
4014{
4015        struct action_data *data;
4016        int ret = -EINVAL;
4017        char *var_str;
4018
4019        data = kzalloc(sizeof(*data), GFP_KERNEL);
4020        if (!data)
4021                return ERR_PTR(-ENOMEM);
4022
4023        var_str = strsep(&str, ")");
4024        if (!var_str || !str) {
4025                ret = -EINVAL;
4026                goto free;
4027        }
4028
4029        data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
4030        if (!data->track_data.var_str) {
4031                ret = -ENOMEM;
4032                goto free;
4033        }
4034
4035        ret = action_parse(hist_data->event_file->tr, str, data, handler);
4036        if (ret)
4037                goto free;
4038 out:
4039        return data;
4040 free:
4041        track_data_destroy(hist_data, data);
4042        data = ERR_PTR(ret);
4043        goto out;
4044}
4045
4046static void onmatch_destroy(struct action_data *data)
4047{
4048        kfree(data->match_data.event);
4049        kfree(data->match_data.event_system);
4050
4051        action_data_destroy(data);
4052}
4053
4054static void destroy_field_var(struct field_var *field_var)
4055{
4056        if (!field_var)
4057                return;
4058
4059        destroy_hist_field(field_var->var, 0);
4060        destroy_hist_field(field_var->val, 0);
4061
4062        kfree(field_var);
4063}
4064
4065static void destroy_field_vars(struct hist_trigger_data *hist_data)
4066{
4067        unsigned int i;
4068
4069        for (i = 0; i < hist_data->n_field_vars; i++)
4070                destroy_field_var(hist_data->field_vars[i]);
4071}
4072
4073static void save_field_var(struct hist_trigger_data *hist_data,
4074                           struct field_var *field_var)
4075{
4076        hist_data->field_vars[hist_data->n_field_vars++] = field_var;
4077
4078        if (field_var->val->flags & HIST_FIELD_FL_STRING)
4079                hist_data->n_field_var_str++;
4080}
4081
4082
4083static int check_synth_field(struct synth_event *event,
4084                             struct hist_field *hist_field,
4085                             unsigned int field_pos)
4086{
4087        struct synth_field *field;
4088
4089        if (field_pos >= event->n_fields)
4090                return -EINVAL;
4091
4092        field = event->fields[field_pos];
4093
4094        if (strcmp(field->type, hist_field->type) != 0)
4095                return -EINVAL;
4096
4097        return 0;
4098}
4099
4100static struct hist_field *
4101trace_action_find_var(struct hist_trigger_data *hist_data,
4102                      struct action_data *data,
4103                      char *system, char *event, char *var)
4104{
4105        struct trace_array *tr = hist_data->event_file->tr;
4106        struct hist_field *hist_field;
4107
4108        var++; /* skip '$' */
4109
4110        hist_field = find_target_event_var(hist_data, system, event, var);
4111        if (!hist_field) {
4112                if (!system && data->handler == HANDLER_ONMATCH) {
4113                        system = data->match_data.event_system;
4114                        event = data->match_data.event;
4115                }
4116
4117                hist_field = find_event_var(hist_data, system, event, var);
4118        }
4119
4120        if (!hist_field)
4121                hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
4122
4123        return hist_field;
4124}
4125
4126static struct hist_field *
4127trace_action_create_field_var(struct hist_trigger_data *hist_data,
4128                              struct action_data *data, char *system,
4129                              char *event, char *var)
4130{
4131        struct hist_field *hist_field = NULL;
4132        struct field_var *field_var;
4133
4134        /*
4135         * First try to create a field var on the target event (the
4136         * currently being defined).  This will create a variable for
4137         * unqualified fields on the target event, or if qualified,
4138         * target fields that have qualified names matching the target.
4139         */
4140        field_var = create_target_field_var(hist_data, system, event, var);
4141
4142        if (field_var && !IS_ERR(field_var)) {
4143                save_field_var(hist_data, field_var);
4144                hist_field = field_var->var;
4145        } else {
4146                field_var = NULL;
4147                /*
4148                 * If no explicit system.event is specfied, default to
4149                 * looking for fields on the onmatch(system.event.xxx)
4150                 * event.
4151                 */
4152                if (!system && data->handler == HANDLER_ONMATCH) {
4153                        system = data->match_data.event_system;
4154                        event = data->match_data.event;
4155                }
4156
4157                /*
4158                 * At this point, we're looking at a field on another
4159                 * event.  Because we can't modify a hist trigger on
4160                 * another event to add a variable for a field, we need
4161                 * to create a new trigger on that event and create the
4162                 * variable at the same time.
4163                 */
4164                hist_field = create_field_var_hist(hist_data, system, event, var);
4165                if (IS_ERR(hist_field))
4166                        goto free;
4167        }
4168 out:
4169        return hist_field;
4170 free:
4171        destroy_field_var(field_var);
4172        hist_field = NULL;
4173        goto out;
4174}
4175
4176static int trace_action_create(struct hist_trigger_data *hist_data,
4177                               struct action_data *data)
4178{
4179        struct trace_array *tr = hist_data->event_file->tr;
4180        char *event_name, *param, *system = NULL;
4181        struct hist_field *hist_field, *var_ref;
4182        unsigned int i, var_ref_idx;
4183        unsigned int field_pos = 0;
4184        struct synth_event *event;
4185        char *synth_event_name;
4186        int ret = 0;
4187
4188        lockdep_assert_held(&event_mutex);
4189
4190        if (data->use_trace_keyword)
4191                synth_event_name = data->synth_event_name;
4192        else
4193                synth_event_name = data->action_name;
4194
4195        event = find_synth_event(synth_event_name);
4196        if (!event) {
4197                hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4198                return -EINVAL;
4199        }
4200
4201        event->ref++;
4202
4203        var_ref_idx = hist_data->n_var_refs;
4204
4205        for (i = 0; i < data->n_params; i++) {
4206                char *p;
4207
4208                p = param = kstrdup(data->params[i], GFP_KERNEL);
4209                if (!param) {
4210                        ret = -ENOMEM;
4211                        goto err;
4212                }
4213
4214                system = strsep(&param, ".");
4215                if (!param) {
4216                        param = (char *)system;
4217                        system = event_name = NULL;
4218                } else {
4219                        event_name = strsep(&param, ".");
4220                        if (!param) {
4221                                kfree(p);
4222                                ret = -EINVAL;
4223                                goto err;
4224                        }
4225                }
4226
4227                if (param[0] == '$')
4228                        hist_field = trace_action_find_var(hist_data, data,
4229                                                           system, event_name,
4230                                                           param);
4231                else
4232                        hist_field = trace_action_create_field_var(hist_data,
4233                                                                   data,
4234                                                                   system,
4235                                                                   event_name,
4236                                                                   param);
4237
4238                if (!hist_field) {
4239                        kfree(p);
4240                        ret = -EINVAL;
4241                        goto err;
4242                }
4243
4244                if (check_synth_field(event, hist_field, field_pos) == 0) {
4245                        var_ref = create_var_ref(hist_data, hist_field,
4246                                                 system, event_name);
4247                        if (!var_ref) {
4248                                kfree(p);
4249                                ret = -ENOMEM;
4250                                goto err;
4251                        }
4252
4253                        field_pos++;
4254                        kfree(p);
4255                        continue;
4256                }
4257
4258                hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4259                kfree(p);
4260                ret = -EINVAL;
4261                goto err;
4262        }
4263
4264        if (field_pos != event->n_fields) {
4265                hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4266                ret = -EINVAL;
4267                goto err;
4268        }
4269
4270        data->synth_event = event;
4271        data->var_ref_idx = var_ref_idx;
4272 out:
4273        return ret;
4274 err:
4275        event->ref--;
4276
4277        goto out;
4278}
4279
4280static int action_create(struct hist_trigger_data *hist_data,
4281                         struct action_data *data)
4282{
4283        struct trace_event_file *file = hist_data->event_file;
4284        struct trace_array *tr = file->tr;
4285        struct track_data *track_data;
4286        struct field_var *field_var;
4287        unsigned int i;
4288        char *param;
4289        int ret = 0;
4290
4291        if (data->action == ACTION_TRACE)
4292                return trace_action_create(hist_data, data);
4293
4294        if (data->action == ACTION_SNAPSHOT) {
4295                track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4296                if (IS_ERR(track_data)) {
4297                        ret = PTR_ERR(track_data);
4298                        goto out;
4299                }
4300
4301                ret = tracing_snapshot_cond_enable(file->tr, track_data,
4302                                                   cond_snapshot_update);
4303                if (ret)
4304                        track_data_free(track_data);
4305
4306                goto out;
4307        }
4308
4309        if (data->action == ACTION_SAVE) {
4310                if (hist_data->n_save_vars) {
4311                        ret = -EEXIST;
4312                        hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4313                        goto out;
4314                }
4315
4316                for (i = 0; i < data->n_params; i++) {
4317                        param = kstrdup(data->params[i], GFP_KERNEL);
4318                        if (!param) {
4319                                ret = -ENOMEM;
4320                                goto out;
4321                        }
4322
4323                        field_var = create_target_field_var(hist_data, NULL, NULL, param);
4324                        if (IS_ERR(field_var)) {
4325                                hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4326                                         errpos(param));
4327                                ret = PTR_ERR(field_var);
4328                                kfree(param);
4329                                goto out;
4330                        }
4331
4332                        hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4333                        if (field_var->val->flags & HIST_FIELD_FL_STRING)
4334                                hist_data->n_save_var_str++;
4335                        kfree(param);
4336                }
4337        }
4338 out:
4339        return ret;
4340}
4341
4342static int onmatch_create(struct hist_trigger_data *hist_data,
4343                          struct action_data *data)
4344{
4345        return action_create(hist_data, data);
4346}
4347
4348static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4349{
4350        char *match_event, *match_event_system;
4351        struct action_data *data;
4352        int ret = -EINVAL;
4353
4354        data = kzalloc(sizeof(*data), GFP_KERNEL);
4355        if (!data)
4356                return ERR_PTR(-ENOMEM);
4357
4358        match_event = strsep(&str, ")");
4359        if (!match_event || !str) {
4360                hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4361                goto free;
4362        }
4363
4364        match_event_system = strsep(&match_event, ".");
4365        if (!match_event) {
4366                hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4367                goto free;
4368        }
4369
4370        if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4371                hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4372                goto free;
4373        }
4374
4375        data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4376        if (!data->match_data.event) {
4377                ret = -ENOMEM;
4378                goto free;
4379        }
4380
4381        data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4382        if (!data->match_data.event_system) {
4383                ret = -ENOMEM;
4384                goto free;
4385        }
4386
4387        ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4388        if (ret)
4389                goto free;
4390 out:
4391        return data;
4392 free:
4393        onmatch_destroy(data);
4394        data = ERR_PTR(ret);
4395        goto out;
4396}
4397
4398static int create_hitcount_val(struct hist_trigger_data *hist_data)
4399{
4400        hist_data->fields[HITCOUNT_IDX] =
4401                create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4402        if (!hist_data->fields[HITCOUNT_IDX])
4403                return -ENOMEM;
4404
4405        hist_data->n_vals++;
4406        hist_data->n_fields++;
4407
4408        if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4409                return -EINVAL;
4410
4411        return 0;
4412}
4413
4414static int __create_val_field(struct hist_trigger_data *hist_data,
4415                              unsigned int val_idx,
4416                              struct trace_event_file *file,
4417                              char *var_name, char *field_str,
4418                              unsigned long flags)
4419{
4420        struct hist_field *hist_field;
4421        int ret = 0;
4422
4423        hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4424        if (IS_ERR(hist_field)) {
4425                ret = PTR_ERR(hist_field);
4426                goto out;
4427        }
4428
4429        hist_data->fields[val_idx] = hist_field;
4430
4431        ++hist_data->n_vals;
4432        ++hist_data->n_fields;
4433
4434        if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4435                ret = -EINVAL;
4436 out:
4437        return ret;
4438}
4439
4440static int create_val_field(struct hist_trigger_data *hist_data,
4441                            unsigned int val_idx,
4442                            struct trace_event_file *file,
4443                            char *field_str)
4444{
4445        if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4446                return -EINVAL;
4447
4448        return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4449}
4450
4451static int create_var_field(struct hist_trigger_data *hist_data,
4452                            unsigned int val_idx,
4453                            struct trace_event_file *file,
4454                            char *var_name, char *expr_str)
4455{
4456        struct trace_array *tr = hist_data->event_file->tr;
4457        unsigned long flags = 0;
4458
4459        if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4460                return -EINVAL;
4461
4462        if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4463                hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4464                return -EINVAL;
4465        }
4466
4467        flags |= HIST_FIELD_FL_VAR;
4468        hist_data->n_vars++;
4469        if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4470                return -EINVAL;
4471
4472        return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4473}
4474
4475static int create_val_fields(struct hist_trigger_data *hist_data,
4476                             struct trace_event_file *file)
4477{
4478        char *fields_str, *field_str;
4479        unsigned int i, j = 1;
4480        int ret;
4481
4482        ret = create_hitcount_val(hist_data);
4483        if (ret)
4484                goto out;
4485
4486        fields_str = hist_data->attrs->vals_str;
4487        if (!fields_str)
4488                goto out;
4489
4490        strsep(&fields_str, "=");
4491        if (!fields_str)
4492                goto out;
4493
4494        for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4495                     j < TRACING_MAP_VALS_MAX; i++) {
4496                field_str = strsep(&fields_str, ",");
4497                if (!field_str)
4498                        break;
4499
4500                if (strcmp(field_str, "hitcount") == 0)
4501                        continue;
4502
4503                ret = create_val_field(hist_data, j++, file, field_str);
4504                if (ret)
4505                        goto out;
4506        }
4507
4508        if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4509                ret = -EINVAL;
4510 out:
4511        return ret;
4512}
4513
4514static int create_key_field(struct hist_trigger_data *hist_data,
4515                            unsigned int key_idx,
4516                            unsigned int key_offset,
4517                            struct trace_event_file *file,
4518                            char *field_str)
4519{
4520        struct trace_array *tr = hist_data->event_file->tr;
4521        struct hist_field *hist_field = NULL;
4522        unsigned long flags = 0;
4523        unsigned int key_size;
4524        int ret = 0;
4525
4526        if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4527                return -EINVAL;
4528
4529        flags |= HIST_FIELD_FL_KEY;
4530
4531        if (strcmp(field_str, "stacktrace") == 0) {
4532                flags |= HIST_FIELD_FL_STACKTRACE;
4533                key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4534                hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4535        } else {
4536                hist_field = parse_expr(hist_data, file, field_str, flags,
4537                                        NULL, 0);
4538                if (IS_ERR(hist_field)) {
4539                        ret = PTR_ERR(hist_field);
4540                        goto out;
4541                }
4542
4543                if (field_has_hist_vars(hist_field, 0)) {
4544                        hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4545                        destroy_hist_field(hist_field, 0);
4546                        ret = -EINVAL;
4547                        goto out;
4548                }
4549
4550                key_size = hist_field->size;
4551        }
4552
4553        hist_data->fields[key_idx] = hist_field;
4554
4555        key_size = ALIGN(key_size, sizeof(u64));
4556        hist_data->fields[key_idx]->size = key_size;
4557        hist_data->fields[key_idx]->offset = key_offset;
4558
4559        hist_data->key_size += key_size;
4560
4561        if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4562                ret = -EINVAL;
4563                goto out;
4564        }
4565
4566        hist_data->n_keys++;
4567        hist_data->n_fields++;
4568
4569        if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4570                return -EINVAL;
4571
4572        ret = key_size;
4573 out:
4574        return ret;
4575}
4576
4577static int create_key_fields(struct hist_trigger_data *hist_data,
4578                             struct trace_event_file *file)
4579{
4580        unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4581        char *fields_str, *field_str;
4582        int ret = -EINVAL;
4583
4584        fields_str = hist_data->attrs->keys_str;
4585        if (!fields_str)
4586                goto out;
4587
4588        strsep(&fields_str, "=");
4589        if (!fields_str)
4590                goto out;
4591
4592        for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4593                field_str = strsep(&fields_str, ",");
4594                if (!field_str)
4595                        break;
4596                ret = create_key_field(hist_data, i, key_offset,
4597                                       file, field_str);
4598                if (ret < 0)
4599                        goto out;
4600                key_offset += ret;
4601        }
4602        if (fields_str) {
4603                ret = -EINVAL;
4604                goto out;
4605        }
4606        ret = 0;
4607 out:
4608        return ret;
4609}
4610
4611static int create_var_fields(struct hist_trigger_data *hist_data,
4612                             struct trace_event_file *file)
4613{
4614        unsigned int i, j = hist_data->n_vals;
4615        int ret = 0;
4616
4617        unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4618
4619        for (i = 0; i < n_vars; i++) {
4620                char *var_name = hist_data->attrs->var_defs.name[i];
4621                char *expr = hist_data->attrs->var_defs.expr[i];
4622
4623                ret = create_var_field(hist_data, j++, file, var_name, expr);
4624                if (ret)
4625                        goto out;
4626        }
4627 out:
4628        return ret;
4629}
4630
4631static void free_var_defs(struct hist_trigger_data *hist_data)
4632{
4633        unsigned int i;
4634
4635        for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4636                kfree(hist_data->attrs->var_defs.name[i]);
4637                kfree(hist_data->attrs->var_defs.expr[i]);
4638        }
4639
4640        hist_data->attrs->var_defs.n_vars = 0;
4641}
4642
4643static int parse_var_defs(struct hist_trigger_data *hist_data)
4644{
4645        struct trace_array *tr = hist_data->event_file->tr;
4646        char *s, *str, *var_name, *field_str;
4647        unsigned int i, j, n_vars = 0;
4648        int ret = 0;
4649
4650        for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4651                str = hist_data->attrs->assignment_str[i];
4652                for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4653                        field_str = strsep(&str, ",");
4654                        if (!field_str)
4655                                break;
4656
4657                        var_name = strsep(&field_str, "=");
4658                        if (!var_name || !field_str) {
4659                                hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4660                                         errpos(var_name));
4661                                ret = -EINVAL;
4662                                goto free;
4663                        }
4664
4665                        if (n_vars == TRACING_MAP_VARS_MAX) {
4666                                hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4667                                ret = -EINVAL;
4668                                goto free;
4669                        }
4670
4671                        s = kstrdup(var_name, GFP_KERNEL);
4672                        if (!s) {
4673                                ret = -ENOMEM;
4674                                goto free;
4675                        }
4676                        hist_data->attrs->var_defs.name[n_vars] = s;
4677
4678                        s = kstrdup(field_str, GFP_KERNEL);
4679                        if (!s) {
4680                                kfree(hist_data->attrs->var_defs.name[n_vars]);
4681                                ret = -ENOMEM;
4682                                goto free;
4683                        }
4684                        hist_data->attrs->var_defs.expr[n_vars++] = s;
4685
4686                        hist_data->attrs->var_defs.n_vars = n_vars;
4687                }
4688        }
4689
4690        return ret;
4691 free:
4692        free_var_defs(hist_data);
4693
4694        return ret;
4695}
4696
4697static int create_hist_fields(struct hist_trigger_data *hist_data,
4698                              struct trace_event_file *file)
4699{
4700        int ret;
4701
4702        ret = parse_var_defs(hist_data);
4703        if (ret)
4704                goto out;
4705
4706        ret = create_val_fields(hist_data, file);
4707        if (ret)
4708                goto out;
4709
4710        ret = create_var_fields(hist_data, file);
4711        if (ret)
4712                goto out;
4713
4714        ret = create_key_fields(hist_data, file);
4715        if (ret)
4716                goto out;
4717 out:
4718        free_var_defs(hist_data);
4719
4720        return ret;
4721}
4722
4723static int is_descending(const char *str)
4724{
4725        if (!str)
4726                return 0;
4727
4728        if (strcmp(str, "descending") == 0)
4729                return 1;
4730
4731        if (strcmp(str, "ascending") == 0)
4732                return 0;
4733
4734        return -EINVAL;
4735}
4736
4737static int create_sort_keys(struct hist_trigger_data *hist_data)
4738{
4739        char *fields_str = hist_data->attrs->sort_key_str;
4740        struct tracing_map_sort_key *sort_key;
4741        int descending, ret = 0;
4742        unsigned int i, j, k;
4743
4744        hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4745
4746        if (!fields_str)
4747                goto out;
4748
4749        strsep(&fields_str, "=");
4750        if (!fields_str) {
4751                ret = -EINVAL;
4752                goto out;
4753        }
4754
4755        for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4756                struct hist_field *hist_field;
4757                char *field_str, *field_name;
4758                const char *test_name;
4759
4760                sort_key = &hist_data->sort_keys[i];
4761
4762                field_str = strsep(&fields_str, ",");
4763                if (!field_str) {
4764                        if (i == 0)
4765                                ret = -EINVAL;
4766                        break;
4767                }
4768
4769                if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4770                        ret = -EINVAL;
4771                        break;
4772                }
4773
4774                field_name = strsep(&field_str, ".");
4775                if (!field_name) {
4776                        ret = -EINVAL;
4777                        break;
4778                }
4779
4780                if (strcmp(field_name, "hitcount") == 0) {
4781                        descending = is_descending(field_str);
4782                        if (descending < 0) {
4783                                ret = descending;
4784                                break;
4785                        }
4786                        sort_key->descending = descending;
4787                        continue;
4788                }
4789
4790                for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4791                        unsigned int idx;
4792
4793                        hist_field = hist_data->fields[j];
4794                        if (hist_field->flags & HIST_FIELD_FL_VAR)
4795                                continue;
4796
4797                        idx = k++;
4798
4799                        test_name = hist_field_name(hist_field, 0);
4800
4801                        if (strcmp(field_name, test_name) == 0) {
4802                                sort_key->field_idx = idx;
4803                                descending = is_descending(field_str);
4804                                if (descending < 0) {
4805                                        ret = descending;
4806                                        goto out;
4807                                }
4808                                sort_key->descending = descending;
4809                                break;
4810                        }
4811                }
4812                if (j == hist_data->n_fields) {
4813                        ret = -EINVAL;
4814                        break;
4815                }
4816        }
4817
4818        hist_data->n_sort_keys = i;
4819 out:
4820        return ret;
4821}
4822
4823static void destroy_actions(struct hist_trigger_data *hist_data)
4824{
4825        unsigned int i;
4826
4827        for (i = 0; i < hist_data->n_actions; i++) {
4828                struct action_data *data = hist_data->actions[i];
4829
4830                if (data->handler == HANDLER_ONMATCH)
4831                        onmatch_destroy(data);
4832                else if (data->handler == HANDLER_ONMAX ||
4833                         data->handler == HANDLER_ONCHANGE)
4834                        track_data_destroy(hist_data, data);
4835                else
4836                        kfree(data);
4837        }
4838}
4839
4840static int parse_actions(struct hist_trigger_data *hist_data)
4841{
4842        struct trace_array *tr = hist_data->event_file->tr;
4843        struct action_data *data;
4844        unsigned int i;
4845        int ret = 0;
4846        char *str;
4847        int len;
4848
4849        for (i = 0; i < hist_data->attrs->n_actions; i++) {
4850                str = hist_data->attrs->action_str[i];
4851
4852                if ((len = str_has_prefix(str, "onmatch("))) {
4853                        char *action_str = str + len;
4854
4855                        data = onmatch_parse(tr, action_str);
4856                        if (IS_ERR(data)) {
4857                                ret = PTR_ERR(data);
4858                                break;
4859                        }
4860                } else if ((len = str_has_prefix(str, "onmax("))) {
4861                        char *action_str = str + len;
4862
4863                        data = track_data_parse(hist_data, action_str,
4864                                                HANDLER_ONMAX);
4865                        if (IS_ERR(data)) {
4866                                ret = PTR_ERR(data);
4867                                break;
4868                        }
4869                } else if ((len = str_has_prefix(str, "onchange("))) {
4870                        char *action_str = str + len;
4871
4872                        data = track_data_parse(hist_data, action_str,
4873                                                HANDLER_ONCHANGE);
4874                        if (IS_ERR(data)) {
4875                                ret = PTR_ERR(data);
4876                                break;
4877                        }
4878                } else {
4879                        ret = -EINVAL;
4880                        break;
4881                }
4882
4883                hist_data->actions[hist_data->n_actions++] = data;
4884        }
4885
4886        return ret;
4887}
4888
4889static int create_actions(struct hist_trigger_data *hist_data)
4890{
4891        struct action_data *data;
4892        unsigned int i;
4893        int ret = 0;
4894
4895        for (i = 0; i < hist_data->attrs->n_actions; i++) {
4896                data = hist_data->actions[i];
4897
4898                if (data->handler == HANDLER_ONMATCH) {
4899                        ret = onmatch_create(hist_data, data);
4900                        if (ret)
4901                                break;
4902                } else if (data->handler == HANDLER_ONMAX ||
4903                           data->handler == HANDLER_ONCHANGE) {
4904                        ret = track_data_create(hist_data, data);
4905                        if (ret)
4906                                break;
4907                } else {
4908                        ret = -EINVAL;
4909                        break;
4910                }
4911        }
4912
4913        return ret;
4914}
4915
4916static void print_actions(struct seq_file *m,
4917                          struct hist_trigger_data *hist_data,
4918                          struct tracing_map_elt *elt)
4919{
4920        unsigned int i;
4921
4922        for (i = 0; i < hist_data->n_actions; i++) {
4923                struct action_data *data = hist_data->actions[i];
4924
4925                if (data->action == ACTION_SNAPSHOT)
4926                        continue;
4927
4928                if (data->handler == HANDLER_ONMAX ||
4929                    data->handler == HANDLER_ONCHANGE)
4930                        track_data_print(m, hist_data, elt, data);
4931        }
4932}
4933
4934static void print_action_spec(struct seq_file *m,
4935                              struct hist_trigger_data *hist_data,
4936                              struct action_data *data)
4937{
4938        unsigned int i;
4939
4940        if (data->action == ACTION_SAVE) {
4941                for (i = 0; i < hist_data->n_save_vars; i++) {
4942                        seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4943                        if (i < hist_data->n_save_vars - 1)
4944                                seq_puts(m, ",");
4945                }
4946        } else if (data->action == ACTION_TRACE) {
4947                if (data->use_trace_keyword)
4948                        seq_printf(m, "%s", data->synth_event_name);
4949                for (i = 0; i < data->n_params; i++) {
4950                        if (i || data->use_trace_keyword)
4951                                seq_puts(m, ",");
4952                        seq_printf(m, "%s", data->params[i]);
4953                }
4954        }
4955}
4956
4957static void print_track_data_spec(struct seq_file *m,
4958                                  struct hist_trigger_data *hist_data,
4959                                  struct action_data *data)
4960{
4961        if (data->handler == HANDLER_ONMAX)
4962                seq_puts(m, ":onmax(");
4963        else if (data->handler == HANDLER_ONCHANGE)
4964                seq_puts(m, ":onchange(");
4965        seq_printf(m, "%s", data->track_data.var_str);
4966        seq_printf(m, ").%s(", data->action_name);
4967
4968        print_action_spec(m, hist_data, data);
4969
4970        seq_puts(m, ")");
4971}
4972
4973static void print_onmatch_spec(struct seq_file *m,
4974                               struct hist_trigger_data *hist_data,
4975                               struct action_data *data)
4976{
4977        seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4978                   data->match_data.event);
4979
4980        seq_printf(m, "%s(", data->action_name);
4981
4982        print_action_spec(m, hist_data, data);
4983
4984        seq_puts(m, ")");
4985}
4986
4987static bool actions_match(struct hist_trigger_data *hist_data,
4988                          struct hist_trigger_data *hist_data_test)
4989{
4990        unsigned int i, j;
4991
4992        if (hist_data->n_actions != hist_data_test->n_actions)
4993                return false;
4994
4995        for (i = 0; i < hist_data->n_actions; i++) {
4996                struct action_data *data = hist_data->actions[i];
4997                struct action_data *data_test = hist_data_test->actions[i];
4998                char *action_name, *action_name_test;
4999
5000                if (data->handler != data_test->handler)
5001                        return false;
5002                if (data->action != data_test->action)
5003                        return false;
5004
5005                if (data->n_params != data_test->n_params)
5006                        return false;
5007
5008                for (j = 0; j < data->n_params; j++) {
5009                        if (strcmp(data->params[j], data_test->params[j]) != 0)
5010                                return false;
5011                }
5012
5013                if (data->use_trace_keyword)
5014                        action_name = data->synth_event_name;
5015                else
5016                        action_name = data->action_name;
5017
5018                if (data_test->use_trace_keyword)
5019                        action_name_test = data_test->synth_event_name;
5020                else
5021                        action_name_test = data_test->action_name;
5022
5023                if (strcmp(action_name, action_name_test) != 0)
5024                        return false;
5025
5026                if (data->handler == HANDLER_ONMATCH) {
5027                        if (strcmp(data->match_data.event_system,
5028                                   data_test->match_data.event_system) != 0)
5029                                return false;
5030                        if (strcmp(data->match_data.event,
5031                                   data_test->match_data.event) != 0)
5032                                return false;
5033                } else if (data->handler == HANDLER_ONMAX ||
5034                           data->handler == HANDLER_ONCHANGE) {
5035                        if (strcmp(data->track_data.var_str,
5036                                   data_test->track_data.var_str) != 0)
5037                                return false;
5038                }
5039        }
5040
5041        return true;
5042}
5043
5044
5045static void print_actions_spec(struct seq_file *m,
5046                               struct hist_trigger_data *hist_data)
5047{
5048        unsigned int i;
5049
5050        for (i = 0; i < hist_data->n_actions; i++) {
5051                struct action_data *data = hist_data->actions[i];
5052
5053                if (data->handler == HANDLER_ONMATCH)
5054                        print_onmatch_spec(m, hist_data, data);
5055                else if (data->handler == HANDLER_ONMAX ||
5056                         data->handler == HANDLER_ONCHANGE)
5057                        print_track_data_spec(m, hist_data, data);
5058        }
5059}
5060
5061static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5062{
5063        unsigned int i;
5064
5065        for (i = 0; i < hist_data->n_field_var_hists; i++) {
5066                kfree(hist_data->field_var_hists[i]->cmd);
5067                kfree(hist_data->field_var_hists[i]);
5068        }
5069}
5070
5071static void destroy_hist_data(struct hist_trigger_data *hist_data)
5072{
5073        if (!hist_data)
5074                return;
5075
5076        destroy_hist_trigger_attrs(hist_data->attrs);
5077        destroy_hist_fields(hist_data);
5078        tracing_map_destroy(hist_data->map);
5079
5080        destroy_actions(hist_data);
5081        destroy_field_vars(hist_data);
5082        destroy_field_var_hists(hist_data);
5083
5084        kfree(hist_data);
5085}
5086
5087static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5088{
5089        struct tracing_map *map = hist_data->map;
5090        struct ftrace_event_field *field;
5091        struct hist_field *hist_field;
5092        int i, idx = 0;
5093
5094        for_each_hist_field(i, hist_data) {
5095                hist_field = hist_data->fields[i];
5096                if (hist_field->flags & HIST_FIELD_FL_KEY) {
5097                        tracing_map_cmp_fn_t cmp_fn;
5098
5099                        field = hist_field->field;
5100
5101                        if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5102                                cmp_fn = tracing_map_cmp_none;
5103                        else if (!field)
5104                                cmp_fn = tracing_map_cmp_num(hist_field->size,
5105                                                             hist_field->is_signed);
5106                        else if (is_string_field(field))
5107                                cmp_fn = tracing_map_cmp_string;
5108                        else
5109                                cmp_fn = tracing_map_cmp_num(field->size,
5110                                                             field->is_signed);
5111                        idx = tracing_map_add_key_field(map,
5112                                                        hist_field->offset,
5113                                                        cmp_fn);
5114                } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5115                        idx = tracing_map_add_sum_field(map);
5116
5117                if (idx < 0)
5118                        return idx;
5119
5120                if (hist_field->flags & HIST_FIELD_FL_VAR) {
5121                        idx = tracing_map_add_var(map);
5122                        if (idx < 0)
5123                                return idx;
5124                        hist_field->var.idx = idx;
5125                        hist_field->var.hist_data = hist_data;
5126                }
5127        }
5128
5129        return 0;
5130}
5131
5132static struct hist_trigger_data *
5133create_hist_data(unsigned int map_bits,
5134                 struct hist_trigger_attrs *attrs,
5135                 struct trace_event_file *file,
5136                 bool remove)
5137{
5138        const struct tracing_map_ops *map_ops = NULL;
5139        struct hist_trigger_data *hist_data;
5140        int ret = 0;
5141
5142        hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5143        if (!hist_data)
5144                return ERR_PTR(-ENOMEM);
5145
5146        hist_data->attrs = attrs;
5147        hist_data->remove = remove;
5148        hist_data->event_file = file;
5149
5150        ret = parse_actions(hist_data);
5151        if (ret)
5152                goto free;
5153
5154        ret = create_hist_fields(hist_data, file);
5155        if (ret)
5156                goto free;
5157
5158        ret = create_sort_keys(hist_data);
5159        if (ret)
5160                goto free;
5161
5162        map_ops = &hist_trigger_elt_data_ops;
5163
5164        hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5165                                            map_ops, hist_data);
5166        if (IS_ERR(hist_data->map)) {
5167                ret = PTR_ERR(hist_data->map);
5168                hist_data->map = NULL;
5169                goto free;
5170        }
5171
5172        ret = create_tracing_map_fields(hist_data);
5173        if (ret)
5174                goto free;
5175 out:
5176        return hist_data;
5177 free:
5178        hist_data->attrs = NULL;
5179
5180        destroy_hist_data(hist_data);
5181
5182        hist_data = ERR_PTR(ret);
5183
5184        goto out;
5185}
5186
5187static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5188                                    struct tracing_map_elt *elt, void *rec,
5189                                    struct ring_buffer_event *rbe,
5190                                    u64 *var_ref_vals)
5191{
5192        struct hist_elt_data *elt_data;
5193        struct hist_field *hist_field;
5194        unsigned int i, var_idx;
5195        u64 hist_val;
5196
5197        elt_data = elt->private_data;
5198        elt_data->var_ref_vals = var_ref_vals;
5199
5200        for_each_hist_val_field(i, hist_data) {
5201                hist_field = hist_data->fields[i];
5202                hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5203                if (hist_field->flags & HIST_FIELD_FL_VAR) {
5204                        var_idx = hist_field->var.idx;
5205                        tracing_map_set_var(elt, var_idx, hist_val);
5206                        continue;
5207                }
5208                tracing_map_update_sum(elt, i, hist_val);
5209        }
5210
5211        for_each_hist_key_field(i, hist_data) {
5212                hist_field = hist_data->fields[i];
5213                if (hist_field->flags & HIST_FIELD_FL_VAR) {
5214                        hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5215                        var_idx = hist_field->var.idx;
5216                        tracing_map_set_var(elt, var_idx, hist_val);
5217                }
5218        }
5219
5220        update_field_vars(hist_data, elt, rbe, rec);
5221}
5222
5223static inline void add_to_key(char *compound_key, void *key,
5224                              struct hist_field *key_field, void *rec)
5225{
5226        size_t size = key_field->size;
5227
5228        if (key_field->flags & HIST_FIELD_FL_STRING) {
5229                struct ftrace_event_field *field;
5230
5231                field = key_field->field;
5232                if (field->filter_type == FILTER_DYN_STRING)
5233                        size = *(u32 *)(rec + field->offset) >> 16;
5234                else if (field->filter_type == FILTER_PTR_STRING)
5235                        size = strlen(key);
5236                else if (field->filter_type == FILTER_STATIC_STRING)
5237                        size = field->size;
5238
5239                /* ensure NULL-termination */
5240                if (size > key_field->size - 1)
5241                        size = key_field->size - 1;
5242
5243                strncpy(compound_key + key_field->offset, (char *)key, size);
5244        } else
5245                memcpy(compound_key + key_field->offset, key, size);
5246}
5247
5248static void
5249hist_trigger_actions(struct hist_trigger_data *hist_data,
5250                     struct tracing_map_elt *elt, void *rec,
5251                     struct ring_buffer_event *rbe, void *key,
5252                     u64 *var_ref_vals)
5253{
5254        struct action_data *data;
5255        unsigned int i;
5256
5257        for (i = 0; i < hist_data->n_actions; i++) {
5258                data = hist_data->actions[i];
5259                data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
5260        }
5261}
5262
5263static void event_hist_trigger(struct event_trigger_data *data, void *rec,
5264                               struct ring_buffer_event *rbe)
5265{
5266        struct hist_trigger_data *hist_data = data->private_data;
5267        bool use_compound_key = (hist_data->n_keys > 1);
5268        unsigned long entries[HIST_STACKTRACE_DEPTH];
5269        u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5270        char compound_key[HIST_KEY_SIZE_MAX];
5271        struct tracing_map_elt *elt = NULL;
5272        struct hist_field *key_field;