1
2
3
4
5
6
7
8#include <linux/module.h>
9#include <linux/kallsyms.h>
10#include <linux/security.h>
11#include <linux/mutex.h>
12#include <linux/slab.h>
13#include <linux/stacktrace.h>
14#include <linux/rculist.h>
15#include <linux/tracefs.h>
16
17
18#include <linux/trace_events.h>
19#include <trace/events/mmflags.h>
20
21#include "tracing_map.h"
22#include "trace_synth.h"
23
24#define ERRORS \
25 C(NONE, "No error"), \
26 C(DUPLICATE_VAR, "Variable already defined"), \
27 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
28 C(TOO_MANY_VARS, "Too many variables defined"), \
29 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \
30 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \
31 C(TRIGGER_EEXIST, "Hist trigger already exists"), \
32 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
33 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \
34 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \
35 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \
36 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \
37 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \
38 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \
39 C(HIST_NOT_FOUND, "Matching event histogram not found"), \
40 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \
41 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \
42 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \
43 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \
44 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
45 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \
46 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \
47 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \
48 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \
49 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \
50 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \
51 C(TOO_MANY_PARAMS, "Too many action params"), \
52 C(PARAM_NOT_FOUND, "Couldn't find param"), \
53 C(INVALID_PARAM, "Invalid action param"), \
54 C(ACTION_NOT_FOUND, "No action found"), \
55 C(NO_SAVE_PARAMS, "No params found for save()"), \
56 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
57 C(ACTION_MISMATCH, "Handler doesn't support action"), \
58 C(NO_CLOSING_PAREN, "No closing paren found"), \
59 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \
60 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \
61 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \
62 C(VAR_NOT_FOUND, "Couldn't find variable"), \
63 C(FIELD_NOT_FOUND, "Couldn't find field"), \
64 C(EMPTY_ASSIGNMENT, "Empty assignment"), \
65 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \
66 C(EMPTY_SORT_FIELD, "Empty sort field"), \
67 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \
68 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), \
69 C(INVALID_STR_OPERAND, "String type can not be an operand in expression"), \
70 C(EXPECT_NUMBER, "Expecting numeric literal"), \
71 C(UNARY_MINUS_SUBEXPR, "Unary minus not supported in sub-expressions"), \
72 C(DIVISION_BY_ZERO, "Division by zero"),
73
74#undef C
75#define C(a, b) HIST_ERR_##a
76
77enum { ERRORS };
78
79#undef C
80#define C(a, b) b
81
82static const char *err_text[] = { ERRORS };
83
84struct hist_field;
85
86typedef u64 (*hist_field_fn_t) (struct hist_field *field,
87 struct tracing_map_elt *elt,
88 struct trace_buffer *buffer,
89 struct ring_buffer_event *rbe,
90 void *event);
91
92#define HIST_FIELD_OPERANDS_MAX 2
93#define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
94#define HIST_ACTIONS_MAX 8
95#define HIST_CONST_DIGITS_MAX 21
96#define HIST_DIV_SHIFT 20
97
98enum field_op_id {
99 FIELD_OP_NONE,
100 FIELD_OP_PLUS,
101 FIELD_OP_MINUS,
102 FIELD_OP_UNARY_MINUS,
103 FIELD_OP_DIV,
104 FIELD_OP_MULT,
105};
106
107
108
109
110
111
112
113
114
115
116
117struct hist_var {
118 char *name;
119 struct hist_trigger_data *hist_data;
120 unsigned int idx;
121};
122
123struct hist_field {
124 struct ftrace_event_field *field;
125 unsigned long flags;
126 hist_field_fn_t fn;
127 unsigned int ref;
128 unsigned int size;
129 unsigned int offset;
130 unsigned int is_signed;
131 unsigned long buckets;
132 const char *type;
133 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
134 struct hist_trigger_data *hist_data;
135
136
137
138
139 struct hist_var var;
140 enum field_op_id operator;
141 char *system;
142 char *event_name;
143
144
145
146
147
148 char *name;
149
150
151
152
153
154
155
156
157
158 unsigned int var_ref_idx;
159 bool read_once;
160
161 unsigned int var_str_idx;
162
163
164 u64 constant;
165
166 u64 div_multiplier;
167};
168
169static u64 hist_field_none(struct hist_field *field,
170 struct tracing_map_elt *elt,
171 struct trace_buffer *buffer,
172 struct ring_buffer_event *rbe,
173 void *event)
174{
175 return 0;
176}
177
178static u64 hist_field_const(struct hist_field *field,
179 struct tracing_map_elt *elt,
180 struct trace_buffer *buffer,
181 struct ring_buffer_event *rbe,
182 void *event)
183{
184 return field->constant;
185}
186
187static u64 hist_field_counter(struct hist_field *field,
188 struct tracing_map_elt *elt,
189 struct trace_buffer *buffer,
190 struct ring_buffer_event *rbe,
191 void *event)
192{
193 return 1;
194}
195
196static u64 hist_field_string(struct hist_field *hist_field,
197 struct tracing_map_elt *elt,
198 struct trace_buffer *buffer,
199 struct ring_buffer_event *rbe,
200 void *event)
201{
202 char *addr = (char *)(event + hist_field->field->offset);
203
204 return (u64)(unsigned long)addr;
205}
206
207static u64 hist_field_dynstring(struct hist_field *hist_field,
208 struct tracing_map_elt *elt,
209 struct trace_buffer *buffer,
210 struct ring_buffer_event *rbe,
211 void *event)
212{
213 u32 str_item = *(u32 *)(event + hist_field->field->offset);
214 int str_loc = str_item & 0xffff;
215 char *addr = (char *)(event + str_loc);
216
217 return (u64)(unsigned long)addr;
218}
219
220static u64 hist_field_reldynstring(struct hist_field *hist_field,
221 struct tracing_map_elt *elt,
222 struct trace_buffer *buffer,
223 struct ring_buffer_event *rbe,
224 void *event)
225{
226 u32 *item = event + hist_field->field->offset;
227 u32 str_item = *item;
228 int str_loc = str_item & 0xffff;
229 char *addr = (char *)&item[1] + str_loc;
230
231 return (u64)(unsigned long)addr;
232}
233
234static u64 hist_field_pstring(struct hist_field *hist_field,
235 struct tracing_map_elt *elt,
236 struct trace_buffer *buffer,
237 struct ring_buffer_event *rbe,
238 void *event)
239{
240 char **addr = (char **)(event + hist_field->field->offset);
241
242 return (u64)(unsigned long)*addr;
243}
244
245static u64 hist_field_log2(struct hist_field *hist_field,
246 struct tracing_map_elt *elt,
247 struct trace_buffer *buffer,
248 struct ring_buffer_event *rbe,
249 void *event)
250{
251 struct hist_field *operand = hist_field->operands[0];
252
253 u64 val = operand->fn(operand, elt, buffer, rbe, event);
254
255 return (u64) ilog2(roundup_pow_of_two(val));
256}
257
258static u64 hist_field_bucket(struct hist_field *hist_field,
259 struct tracing_map_elt *elt,
260 struct trace_buffer *buffer,
261 struct ring_buffer_event *rbe,
262 void *event)
263{
264 struct hist_field *operand = hist_field->operands[0];
265 unsigned long buckets = hist_field->buckets;
266
267 u64 val = operand->fn(operand, elt, buffer, rbe, event);
268
269 if (WARN_ON_ONCE(!buckets))
270 return val;
271
272 if (val >= LONG_MAX)
273 val = div64_ul(val, buckets);
274 else
275 val = (u64)((unsigned long)val / buckets);
276 return val * buckets;
277}
278
279static u64 hist_field_plus(struct hist_field *hist_field,
280 struct tracing_map_elt *elt,
281 struct trace_buffer *buffer,
282 struct ring_buffer_event *rbe,
283 void *event)
284{
285 struct hist_field *operand1 = hist_field->operands[0];
286 struct hist_field *operand2 = hist_field->operands[1];
287
288 u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
289 u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
290
291 return val1 + val2;
292}
293
294static u64 hist_field_minus(struct hist_field *hist_field,
295 struct tracing_map_elt *elt,
296 struct trace_buffer *buffer,
297 struct ring_buffer_event *rbe,
298 void *event)
299{
300 struct hist_field *operand1 = hist_field->operands[0];
301 struct hist_field *operand2 = hist_field->operands[1];
302
303 u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
304 u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
305
306 return val1 - val2;
307}
308
309static u64 hist_field_div(struct hist_field *hist_field,
310 struct tracing_map_elt *elt,
311 struct trace_buffer *buffer,
312 struct ring_buffer_event *rbe,
313 void *event)
314{
315 struct hist_field *operand1 = hist_field->operands[0];
316 struct hist_field *operand2 = hist_field->operands[1];
317
318 u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
319 u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
320
321
322 if (!val2)
323 return -1;
324
325
326 if (!(val2 & (val2 - 1)))
327 return val1 >> __ffs64(val2);
328
329 return div64_u64(val1, val2);
330}
331
332static u64 div_by_power_of_two(struct hist_field *hist_field,
333 struct tracing_map_elt *elt,
334 struct trace_buffer *buffer,
335 struct ring_buffer_event *rbe,
336 void *event)
337{
338 struct hist_field *operand1 = hist_field->operands[0];
339 struct hist_field *operand2 = hist_field->operands[1];
340
341 u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
342
343 return val1 >> __ffs64(operand2->constant);
344}
345
346static u64 div_by_not_power_of_two(struct hist_field *hist_field,
347 struct tracing_map_elt *elt,
348 struct trace_buffer *buffer,
349 struct ring_buffer_event *rbe,
350 void *event)
351{
352 struct hist_field *operand1 = hist_field->operands[0];
353 struct hist_field *operand2 = hist_field->operands[1];
354
355 u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
356
357 return div64_u64(val1, operand2->constant);
358}
359
360static u64 div_by_mult_and_shift(struct hist_field *hist_field,
361 struct tracing_map_elt *elt,
362 struct trace_buffer *buffer,
363 struct ring_buffer_event *rbe,
364 void *event)
365{
366 struct hist_field *operand1 = hist_field->operands[0];
367 struct hist_field *operand2 = hist_field->operands[1];
368
369 u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385 if (val1 < (1 << HIST_DIV_SHIFT)) {
386 u64 mult = operand2->div_multiplier;
387
388 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
389 }
390
391 return div64_u64(val1, operand2->constant);
392}
393
394static u64 hist_field_mult(struct hist_field *hist_field,
395 struct tracing_map_elt *elt,
396 struct trace_buffer *buffer,
397 struct ring_buffer_event *rbe,
398 void *event)
399{
400 struct hist_field *operand1 = hist_field->operands[0];
401 struct hist_field *operand2 = hist_field->operands[1];
402
403 u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
404 u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
405
406 return val1 * val2;
407}
408
409static u64 hist_field_unary_minus(struct hist_field *hist_field,
410 struct tracing_map_elt *elt,
411 struct trace_buffer *buffer,
412 struct ring_buffer_event *rbe,
413 void *event)
414{
415 struct hist_field *operand = hist_field->operands[0];
416
417 s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
418 u64 val = (u64)-sval;
419
420 return val;
421}
422
423#define DEFINE_HIST_FIELD_FN(type) \
424 static u64 hist_field_##type(struct hist_field *hist_field, \
425 struct tracing_map_elt *elt, \
426 struct trace_buffer *buffer, \
427 struct ring_buffer_event *rbe, \
428 void *event) \
429{ \
430 type *addr = (type *)(event + hist_field->field->offset); \
431 \
432 return (u64)(unsigned long)*addr; \
433}
434
435DEFINE_HIST_FIELD_FN(s64);
436DEFINE_HIST_FIELD_FN(u64);
437DEFINE_HIST_FIELD_FN(s32);
438DEFINE_HIST_FIELD_FN(u32);
439DEFINE_HIST_FIELD_FN(s16);
440DEFINE_HIST_FIELD_FN(u16);
441DEFINE_HIST_FIELD_FN(s8);
442DEFINE_HIST_FIELD_FN(u8);
443
444#define for_each_hist_field(i, hist_data) \
445 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
446
447#define for_each_hist_val_field(i, hist_data) \
448 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
449
450#define for_each_hist_key_field(i, hist_data) \
451 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
452
453#define HIST_STACKTRACE_DEPTH 16
454#define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
455#define HIST_STACKTRACE_SKIP 5
456
457#define HITCOUNT_IDX 0
458#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
459
460enum hist_field_flags {
461 HIST_FIELD_FL_HITCOUNT = 1 << 0,
462 HIST_FIELD_FL_KEY = 1 << 1,
463 HIST_FIELD_FL_STRING = 1 << 2,
464 HIST_FIELD_FL_HEX = 1 << 3,
465 HIST_FIELD_FL_SYM = 1 << 4,
466 HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
467 HIST_FIELD_FL_EXECNAME = 1 << 6,
468 HIST_FIELD_FL_SYSCALL = 1 << 7,
469 HIST_FIELD_FL_STACKTRACE = 1 << 8,
470 HIST_FIELD_FL_LOG2 = 1 << 9,
471 HIST_FIELD_FL_TIMESTAMP = 1 << 10,
472 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
473 HIST_FIELD_FL_VAR = 1 << 12,
474 HIST_FIELD_FL_EXPR = 1 << 13,
475 HIST_FIELD_FL_VAR_REF = 1 << 14,
476 HIST_FIELD_FL_CPU = 1 << 15,
477 HIST_FIELD_FL_ALIAS = 1 << 16,
478 HIST_FIELD_FL_BUCKET = 1 << 17,
479 HIST_FIELD_FL_CONST = 1 << 18,
480};
481
482struct var_defs {
483 unsigned int n_vars;
484 char *name[TRACING_MAP_VARS_MAX];
485 char *expr[TRACING_MAP_VARS_MAX];
486};
487
488struct hist_trigger_attrs {
489 char *keys_str;
490 char *vals_str;
491 char *sort_key_str;
492 char *name;
493 char *clock;
494 bool pause;
495 bool cont;
496 bool clear;
497 bool ts_in_usecs;
498 unsigned int map_bits;
499
500 char *assignment_str[TRACING_MAP_VARS_MAX];
501 unsigned int n_assignments;
502
503 char *action_str[HIST_ACTIONS_MAX];
504 unsigned int n_actions;
505
506 struct var_defs var_defs;
507};
508
509struct field_var {
510 struct hist_field *var;
511 struct hist_field *val;
512};
513
514struct field_var_hist {
515 struct hist_trigger_data *hist_data;
516 char *cmd;
517};
518
519struct hist_trigger_data {
520 struct hist_field *fields[HIST_FIELDS_MAX];
521 unsigned int n_vals;
522 unsigned int n_keys;
523 unsigned int n_fields;
524 unsigned int n_vars;
525 unsigned int n_var_str;
526 unsigned int key_size;
527 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
528 unsigned int n_sort_keys;
529 struct trace_event_file *event_file;
530 struct hist_trigger_attrs *attrs;
531 struct tracing_map *map;
532 bool enable_timestamps;
533 bool remove;
534 struct hist_field *var_refs[TRACING_MAP_VARS_MAX];
535 unsigned int n_var_refs;
536
537 struct action_data *actions[HIST_ACTIONS_MAX];
538 unsigned int n_actions;
539
540 struct field_var *field_vars[SYNTH_FIELDS_MAX];
541 unsigned int n_field_vars;
542 unsigned int n_field_var_str;
543 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX];
544 unsigned int n_field_var_hists;
545
546 struct field_var *save_vars[SYNTH_FIELDS_MAX];
547 unsigned int n_save_vars;
548 unsigned int n_save_var_str;
549};
550
551struct action_data;
552
553typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
554 struct tracing_map_elt *elt,
555 struct trace_buffer *buffer, void *rec,
556 struct ring_buffer_event *rbe, void *key,
557 struct action_data *data, u64 *var_ref_vals);
558
559typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
560
561enum handler_id {
562 HANDLER_ONMATCH = 1,
563 HANDLER_ONMAX,
564 HANDLER_ONCHANGE,
565};
566
567enum action_id {
568 ACTION_SAVE = 1,
569 ACTION_TRACE,
570 ACTION_SNAPSHOT,
571};
572
573struct action_data {
574 enum handler_id handler;
575 enum action_id action;
576 char *action_name;
577 action_fn_t fn;
578
579 unsigned int n_params;
580 char *params[SYNTH_FIELDS_MAX];
581
582
583
584
585
586
587
588
589
590
591 unsigned int var_ref_idx[TRACING_MAP_VARS_MAX];
592 struct synth_event *synth_event;
593 bool use_trace_keyword;
594 char *synth_event_name;
595
596 union {
597 struct {
598 char *event;
599 char *event_system;
600 } match_data;
601
602 struct {
603
604
605
606
607
608
609
610
611 char *var_str;
612
613
614
615
616
617 struct hist_field *var_ref;
618
619
620
621
622
623
624 struct hist_field *track_var;
625
626 check_track_val_fn_t check_val;
627 action_fn_t save_data;
628 } track_data;
629 };
630};
631
632struct track_data {
633 u64 track_val;
634 bool updated;
635
636 unsigned int key_len;
637 void *key;
638 struct tracing_map_elt elt;
639
640 struct action_data *action_data;
641 struct hist_trigger_data *hist_data;
642};
643
644struct hist_elt_data {
645 char *comm;
646 u64 *var_ref_vals;
647 char **field_var_str;
648 int n_field_var_str;
649};
650
651struct snapshot_context {
652 struct tracing_map_elt *elt;
653 void *key;
654};
655
656
657
658
659
660static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)
661{
662 u64 div = divisor->constant;
663
664 if (!(div & (div - 1)))
665 return div_by_power_of_two;
666
667
668 if (div > (1 << HIST_DIV_SHIFT))
669 return div_by_not_power_of_two;
670
671 divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
672 return div_by_mult_and_shift;
673}
674
675static void track_data_free(struct track_data *track_data)
676{
677 struct hist_elt_data *elt_data;
678
679 if (!track_data)
680 return;
681
682 kfree(track_data->key);
683
684 elt_data = track_data->elt.private_data;
685 if (elt_data) {
686 kfree(elt_data->comm);
687 kfree(elt_data);
688 }
689
690 kfree(track_data);
691}
692
693static struct track_data *track_data_alloc(unsigned int key_len,
694 struct action_data *action_data,
695 struct hist_trigger_data *hist_data)
696{
697 struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
698 struct hist_elt_data *elt_data;
699
700 if (!data)
701 return ERR_PTR(-ENOMEM);
702
703 data->key = kzalloc(key_len, GFP_KERNEL);
704 if (!data->key) {
705 track_data_free(data);
706 return ERR_PTR(-ENOMEM);
707 }
708
709 data->key_len = key_len;
710 data->action_data = action_data;
711 data->hist_data = hist_data;
712
713 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
714 if (!elt_data) {
715 track_data_free(data);
716 return ERR_PTR(-ENOMEM);
717 }
718
719 data->elt.private_data = elt_data;
720
721 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
722 if (!elt_data->comm) {
723 track_data_free(data);
724 return ERR_PTR(-ENOMEM);
725 }
726
727 return data;
728}
729
730static char last_cmd[MAX_FILTER_STR_VAL];
731static char last_cmd_loc[MAX_FILTER_STR_VAL];
732
733static int errpos(char *str)
734{
735 return err_pos(last_cmd, str);
736}
737
738static void last_cmd_set(struct trace_event_file *file, char *str)
739{
740 const char *system = NULL, *name = NULL;
741 struct trace_event_call *call;
742
743 if (!str)
744 return;
745
746 strcpy(last_cmd, "hist:");
747 strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
748
749 if (file) {
750 call = file->event_call;
751 system = call->class->system;
752 if (system) {
753 name = trace_event_name(call);
754 if (!name)
755 system = NULL;
756 }
757 }
758
759 if (system)
760 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
761}
762
763static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
764{
765 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
766 err_type, err_pos);
767}
768
769static void hist_err_clear(void)
770{
771 last_cmd[0] = '\0';
772 last_cmd_loc[0] = '\0';
773}
774
775typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
776 unsigned int *var_ref_idx);
777
778static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
779 unsigned int *var_ref_idx)
780{
781 struct tracepoint *tp = event->tp;
782
783 if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
784 struct tracepoint_func *probe_func_ptr;
785 synth_probe_func_t probe_func;
786 void *__data;
787
788 if (!(cpu_online(raw_smp_processor_id())))
789 return;
790
791 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
792 if (probe_func_ptr) {
793 do {
794 probe_func = probe_func_ptr->func;
795 __data = probe_func_ptr->data;
796 probe_func(__data, var_ref_vals, var_ref_idx);
797 } while ((++probe_func_ptr)->func);
798 }
799 }
800}
801
802static void action_trace(struct hist_trigger_data *hist_data,
803 struct tracing_map_elt *elt,
804 struct trace_buffer *buffer, void *rec,
805 struct ring_buffer_event *rbe, void *key,
806 struct action_data *data, u64 *var_ref_vals)
807{
808 struct synth_event *event = data->synth_event;
809
810 trace_synth(event, var_ref_vals, data->var_ref_idx);
811}
812
813struct hist_var_data {
814 struct list_head list;
815 struct hist_trigger_data *hist_data;
816};
817
818static u64 hist_field_timestamp(struct hist_field *hist_field,
819 struct tracing_map_elt *elt,
820 struct trace_buffer *buffer,
821 struct ring_buffer_event *rbe,
822 void *event)
823{
824 struct hist_trigger_data *hist_data = hist_field->hist_data;
825 struct trace_array *tr = hist_data->event_file->tr;
826
827 u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
828
829 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
830 ts = ns2usecs(ts);
831
832 return ts;
833}
834
835static u64 hist_field_cpu(struct hist_field *hist_field,
836 struct tracing_map_elt *elt,
837 struct trace_buffer *buffer,
838 struct ring_buffer_event *rbe,
839 void *event)
840{
841 int cpu = smp_processor_id();
842
843 return cpu;
844}
845
846
847
848
849
850
851
852
853
854
855
856
857static struct hist_field *
858check_field_for_var_ref(struct hist_field *hist_field,
859 struct hist_trigger_data *var_data,
860 unsigned int var_idx)
861{
862 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
863
864 if (hist_field && hist_field->var.idx == var_idx &&
865 hist_field->var.hist_data == var_data)
866 return hist_field;
867
868 return NULL;
869}
870
871
872
873
874
875
876
877
878
879
880
881
882
883static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
884 struct hist_trigger_data *var_data,
885 unsigned int var_idx)
886{
887 struct hist_field *hist_field;
888 unsigned int i;
889
890 for (i = 0; i < hist_data->n_var_refs; i++) {
891 hist_field = hist_data->var_refs[i];
892 if (check_field_for_var_ref(hist_field, var_data, var_idx))
893 return hist_field;
894 }
895
896 return NULL;
897}
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
914 unsigned int var_idx)
915{
916 struct trace_array *tr = hist_data->event_file->tr;
917 struct hist_field *found = NULL;
918 struct hist_var_data *var_data;
919
920 list_for_each_entry(var_data, &tr->hist_vars, list) {
921 if (var_data->hist_data == hist_data)
922 continue;
923 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
924 if (found)
925 break;
926 }
927
928 return found;
929}
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944static bool check_var_refs(struct hist_trigger_data *hist_data)
945{
946 struct hist_field *field;
947 bool found = false;
948 int i;
949
950 for_each_hist_field(i, hist_data) {
951 field = hist_data->fields[i];
952 if (field && field->flags & HIST_FIELD_FL_VAR) {
953 if (find_any_var_ref(hist_data, field->var.idx)) {
954 found = true;
955 break;
956 }
957 }
958 }
959
960 return found;
961}
962
963static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
964{
965 struct trace_array *tr = hist_data->event_file->tr;
966 struct hist_var_data *var_data, *found = NULL;
967
968 list_for_each_entry(var_data, &tr->hist_vars, list) {
969 if (var_data->hist_data == hist_data) {
970 found = var_data;
971 break;
972 }
973 }
974
975 return found;
976}
977
978static bool field_has_hist_vars(struct hist_field *hist_field,
979 unsigned int level)
980{
981 int i;
982
983 if (level > 3)
984 return false;
985
986 if (!hist_field)
987 return false;
988
989 if (hist_field->flags & HIST_FIELD_FL_VAR ||
990 hist_field->flags & HIST_FIELD_FL_VAR_REF)
991 return true;
992
993 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
994 struct hist_field *operand;
995
996 operand = hist_field->operands[i];
997 if (field_has_hist_vars(operand, level + 1))
998 return true;
999 }
1000
1001 return false;
1002}
1003
1004static bool has_hist_vars(struct hist_trigger_data *hist_data)
1005{
1006 struct hist_field *hist_field;
1007 int i;
1008
1009 for_each_hist_field(i, hist_data) {
1010 hist_field = hist_data->fields[i];
1011 if (field_has_hist_vars(hist_field, 0))
1012 return true;
1013 }
1014
1015 return false;
1016}
1017
1018static int save_hist_vars(struct hist_trigger_data *hist_data)
1019{
1020 struct trace_array *tr = hist_data->event_file->tr;
1021 struct hist_var_data *var_data;
1022
1023 var_data = find_hist_vars(hist_data);
1024 if (var_data)
1025 return 0;
1026
1027 if (tracing_check_open_get_tr(tr))
1028 return -ENODEV;
1029
1030 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1031 if (!var_data) {
1032 trace_array_put(tr);
1033 return -ENOMEM;
1034 }
1035
1036 var_data->hist_data = hist_data;
1037 list_add(&var_data->list, &tr->hist_vars);
1038
1039 return 0;
1040}
1041
1042static void remove_hist_vars(struct hist_trigger_data *hist_data)
1043{
1044 struct trace_array *tr = hist_data->event_file->tr;
1045 struct hist_var_data *var_data;
1046
1047 var_data = find_hist_vars(hist_data);
1048 if (!var_data)
1049 return;
1050
1051 if (WARN_ON(check_var_refs(hist_data)))
1052 return;
1053
1054 list_del(&var_data->list);
1055
1056 kfree(var_data);
1057
1058 trace_array_put(tr);
1059}
1060
1061static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1062 const char *var_name)
1063{
1064 struct hist_field *hist_field, *found = NULL;
1065 int i;
1066
1067 for_each_hist_field(i, hist_data) {
1068 hist_field = hist_data->fields[i];
1069 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1070 strcmp(hist_field->var.name, var_name) == 0) {
1071 found = hist_field;
1072 break;
1073 }
1074 }
1075
1076 return found;
1077}
1078
1079static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1080 struct trace_event_file *file,
1081 const char *var_name)
1082{
1083 struct hist_trigger_data *test_data;
1084 struct event_trigger_data *test;
1085 struct hist_field *hist_field;
1086
1087 lockdep_assert_held(&event_mutex);
1088
1089 hist_field = find_var_field(hist_data, var_name);
1090 if (hist_field)
1091 return hist_field;
1092
1093 list_for_each_entry(test, &file->triggers, list) {
1094 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1095 test_data = test->private_data;
1096 hist_field = find_var_field(test_data, var_name);
1097 if (hist_field)
1098 return hist_field;
1099 }
1100 }
1101
1102 return NULL;
1103}
1104
1105static struct trace_event_file *find_var_file(struct trace_array *tr,
1106 char *system,
1107 char *event_name,
1108 char *var_name)
1109{
1110 struct hist_trigger_data *var_hist_data;
1111 struct hist_var_data *var_data;
1112 struct trace_event_file *file, *found = NULL;
1113
1114 if (system)
1115 return find_event_file(tr, system, event_name);
1116
1117 list_for_each_entry(var_data, &tr->hist_vars, list) {
1118 var_hist_data = var_data->hist_data;
1119 file = var_hist_data->event_file;
1120 if (file == found)
1121 continue;
1122
1123 if (find_var_field(var_hist_data, var_name)) {
1124 if (found) {
1125 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1126 return NULL;
1127 }
1128
1129 found = file;
1130 }
1131 }
1132
1133 return found;
1134}
1135
1136static struct hist_field *find_file_var(struct trace_event_file *file,
1137 const char *var_name)
1138{
1139 struct hist_trigger_data *test_data;
1140 struct event_trigger_data *test;
1141 struct hist_field *hist_field;
1142
1143 lockdep_assert_held(&event_mutex);
1144
1145 list_for_each_entry(test, &file->triggers, list) {
1146 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1147 test_data = test->private_data;
1148 hist_field = find_var_field(test_data, var_name);
1149 if (hist_field)
1150 return hist_field;
1151 }
1152 }
1153
1154 return NULL;
1155}
1156
1157static struct hist_field *
1158find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1159{
1160 struct trace_array *tr = hist_data->event_file->tr;
1161 struct hist_field *hist_field, *found = NULL;
1162 struct trace_event_file *file;
1163 unsigned int i;
1164
1165 for (i = 0; i < hist_data->n_actions; i++) {
1166 struct action_data *data = hist_data->actions[i];
1167
1168 if (data->handler == HANDLER_ONMATCH) {
1169 char *system = data->match_data.event_system;
1170 char *event_name = data->match_data.event;
1171
1172 file = find_var_file(tr, system, event_name, var_name);
1173 if (!file)
1174 continue;
1175 hist_field = find_file_var(file, var_name);
1176 if (hist_field) {
1177 if (found) {
1178 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1179 errpos(var_name));
1180 return ERR_PTR(-EINVAL);
1181 }
1182
1183 found = hist_field;
1184 }
1185 }
1186 }
1187 return found;
1188}
1189
1190static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1191 char *system,
1192 char *event_name,
1193 char *var_name)
1194{
1195 struct trace_array *tr = hist_data->event_file->tr;
1196 struct hist_field *hist_field = NULL;
1197 struct trace_event_file *file;
1198
1199 if (!system || !event_name) {
1200 hist_field = find_match_var(hist_data, var_name);
1201 if (IS_ERR(hist_field))
1202 return NULL;
1203 if (hist_field)
1204 return hist_field;
1205 }
1206
1207 file = find_var_file(tr, system, event_name, var_name);
1208 if (!file)
1209 return NULL;
1210
1211 hist_field = find_file_var(file, var_name);
1212
1213 return hist_field;
1214}
1215
1216static u64 hist_field_var_ref(struct hist_field *hist_field,
1217 struct tracing_map_elt *elt,
1218 struct trace_buffer *buffer,
1219 struct ring_buffer_event *rbe,
1220 void *event)
1221{
1222 struct hist_elt_data *elt_data;
1223 u64 var_val = 0;
1224
1225 if (WARN_ON_ONCE(!elt))
1226 return var_val;
1227
1228 elt_data = elt->private_data;
1229 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1230
1231 return var_val;
1232}
1233
1234static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1235 u64 *var_ref_vals, bool self)
1236{
1237 struct hist_trigger_data *var_data;
1238 struct tracing_map_elt *var_elt;
1239 struct hist_field *hist_field;
1240 unsigned int i, var_idx;
1241 bool resolved = true;
1242 u64 var_val = 0;
1243
1244 for (i = 0; i < hist_data->n_var_refs; i++) {
1245 hist_field = hist_data->var_refs[i];
1246 var_idx = hist_field->var.idx;
1247 var_data = hist_field->var.hist_data;
1248
1249 if (var_data == NULL) {
1250 resolved = false;
1251 break;
1252 }
1253
1254 if ((self && var_data != hist_data) ||
1255 (!self && var_data == hist_data))
1256 continue;
1257
1258 var_elt = tracing_map_lookup(var_data->map, key);
1259 if (!var_elt) {
1260 resolved = false;
1261 break;
1262 }
1263
1264 if (!tracing_map_var_set(var_elt, var_idx)) {
1265 resolved = false;
1266 break;
1267 }
1268
1269 if (self || !hist_field->read_once)
1270 var_val = tracing_map_read_var(var_elt, var_idx);
1271 else
1272 var_val = tracing_map_read_var_once(var_elt, var_idx);
1273
1274 var_ref_vals[i] = var_val;
1275 }
1276
1277 return resolved;
1278}
1279
1280static const char *hist_field_name(struct hist_field *field,
1281 unsigned int level)
1282{
1283 const char *field_name = "";
1284
1285 if (level > 1)
1286 return field_name;
1287
1288 if (field->field)
1289 field_name = field->field->name;
1290 else if (field->flags & HIST_FIELD_FL_LOG2 ||
1291 field->flags & HIST_FIELD_FL_ALIAS ||
1292 field->flags & HIST_FIELD_FL_BUCKET)
1293 field_name = hist_field_name(field->operands[0], ++level);
1294 else if (field->flags & HIST_FIELD_FL_CPU)
1295 field_name = "common_cpu";
1296 else if (field->flags & HIST_FIELD_FL_EXPR ||
1297 field->flags & HIST_FIELD_FL_VAR_REF) {
1298 if (field->system) {
1299 static char full_name[MAX_FILTER_STR_VAL];
1300
1301 strcat(full_name, field->system);
1302 strcat(full_name, ".");
1303 strcat(full_name, field->event_name);
1304 strcat(full_name, ".");
1305 strcat(full_name, field->name);
1306 field_name = full_name;
1307 } else
1308 field_name = field->name;
1309 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1310 field_name = "common_timestamp";
1311
1312 if (field_name == NULL)
1313 field_name = "";
1314
1315 return field_name;
1316}
1317
1318static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1319{
1320 hist_field_fn_t fn = NULL;
1321
1322 switch (field_size) {
1323 case 8:
1324 if (field_is_signed)
1325 fn = hist_field_s64;
1326 else
1327 fn = hist_field_u64;
1328 break;
1329 case 4:
1330 if (field_is_signed)
1331 fn = hist_field_s32;
1332 else
1333 fn = hist_field_u32;
1334 break;
1335 case 2:
1336 if (field_is_signed)
1337 fn = hist_field_s16;
1338 else
1339 fn = hist_field_u16;
1340 break;
1341 case 1:
1342 if (field_is_signed)
1343 fn = hist_field_s8;
1344 else
1345 fn = hist_field_u8;
1346 break;
1347 }
1348
1349 return fn;
1350}
1351
1352static int parse_map_size(char *str)
1353{
1354 unsigned long size, map_bits;
1355 int ret;
1356
1357 ret = kstrtoul(str, 0, &size);
1358 if (ret)
1359 goto out;
1360
1361 map_bits = ilog2(roundup_pow_of_two(size));
1362 if (map_bits < TRACING_MAP_BITS_MIN ||
1363 map_bits > TRACING_MAP_BITS_MAX)
1364 ret = -EINVAL;
1365 else
1366 ret = map_bits;
1367 out:
1368 return ret;
1369}
1370
1371static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1372{
1373 unsigned int i;
1374
1375 if (!attrs)
1376 return;
1377
1378 for (i = 0; i < attrs->n_assignments; i++)
1379 kfree(attrs->assignment_str[i]);
1380
1381 for (i = 0; i < attrs->n_actions; i++)
1382 kfree(attrs->action_str[i]);
1383
1384 kfree(attrs->name);
1385 kfree(attrs->sort_key_str);
1386 kfree(attrs->keys_str);
1387 kfree(attrs->vals_str);
1388 kfree(attrs->clock);
1389 kfree(attrs);
1390}
1391
1392static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1393{
1394 int ret = -EINVAL;
1395
1396 if (attrs->n_actions >= HIST_ACTIONS_MAX)
1397 return ret;
1398
1399 if ((str_has_prefix(str, "onmatch(")) ||
1400 (str_has_prefix(str, "onmax(")) ||
1401 (str_has_prefix(str, "onchange("))) {
1402 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1403 if (!attrs->action_str[attrs->n_actions]) {
1404 ret = -ENOMEM;
1405 return ret;
1406 }
1407 attrs->n_actions++;
1408 ret = 0;
1409 }
1410 return ret;
1411}
1412
1413static int parse_assignment(struct trace_array *tr,
1414 char *str, struct hist_trigger_attrs *attrs)
1415{
1416 int len, ret = 0;
1417
1418 if ((len = str_has_prefix(str, "key=")) ||
1419 (len = str_has_prefix(str, "keys="))) {
1420 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1421 if (!attrs->keys_str) {
1422 ret = -ENOMEM;
1423 goto out;
1424 }
1425 } else if ((len = str_has_prefix(str, "val=")) ||
1426 (len = str_has_prefix(str, "vals=")) ||
1427 (len = str_has_prefix(str, "values="))) {
1428 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1429 if (!attrs->vals_str) {
1430 ret = -ENOMEM;
1431 goto out;
1432 }
1433 } else if ((len = str_has_prefix(str, "sort="))) {
1434 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1435 if (!attrs->sort_key_str) {
1436 ret = -ENOMEM;
1437 goto out;
1438 }
1439 } else if (str_has_prefix(str, "name=")) {
1440 attrs->name = kstrdup(str, GFP_KERNEL);
1441 if (!attrs->name) {
1442 ret = -ENOMEM;
1443 goto out;
1444 }
1445 } else if ((len = str_has_prefix(str, "clock="))) {
1446 str += len;
1447
1448 str = strstrip(str);
1449 attrs->clock = kstrdup(str, GFP_KERNEL);
1450 if (!attrs->clock) {
1451 ret = -ENOMEM;
1452 goto out;
1453 }
1454 } else if ((len = str_has_prefix(str, "size="))) {
1455 int map_bits = parse_map_size(str + len);
1456
1457 if (map_bits < 0) {
1458 ret = map_bits;
1459 goto out;
1460 }
1461 attrs->map_bits = map_bits;
1462 } else {
1463 char *assignment;
1464
1465 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1466 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1467 ret = -EINVAL;
1468 goto out;
1469 }
1470
1471 assignment = kstrdup(str, GFP_KERNEL);
1472 if (!assignment) {
1473 ret = -ENOMEM;
1474 goto out;
1475 }
1476
1477 attrs->assignment_str[attrs->n_assignments++] = assignment;
1478 }
1479 out:
1480 return ret;
1481}
1482
1483static struct hist_trigger_attrs *
1484parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1485{
1486 struct hist_trigger_attrs *attrs;
1487 int ret = 0;
1488
1489 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1490 if (!attrs)
1491 return ERR_PTR(-ENOMEM);
1492
1493 while (trigger_str) {
1494 char *str = strsep(&trigger_str, ":");
1495 char *rhs;
1496
1497 rhs = strchr(str, '=');
1498 if (rhs) {
1499 if (!strlen(++rhs)) {
1500 ret = -EINVAL;
1501 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1502 goto free;
1503 }
1504 ret = parse_assignment(tr, str, attrs);
1505 if (ret)
1506 goto free;
1507 } else if (strcmp(str, "pause") == 0)
1508 attrs->pause = true;
1509 else if ((strcmp(str, "cont") == 0) ||
1510 (strcmp(str, "continue") == 0))
1511 attrs->cont = true;
1512 else if (strcmp(str, "clear") == 0)
1513 attrs->clear = true;
1514 else {
1515 ret = parse_action(str, attrs);
1516 if (ret)
1517 goto free;
1518 }
1519 }
1520
1521 if (!attrs->keys_str) {
1522 ret = -EINVAL;
1523 goto free;
1524 }
1525
1526 if (!attrs->clock) {
1527 attrs->clock = kstrdup("global", GFP_KERNEL);
1528 if (!attrs->clock) {
1529 ret = -ENOMEM;
1530 goto free;
1531 }
1532 }
1533
1534 return attrs;
1535 free:
1536 destroy_hist_trigger_attrs(attrs);
1537
1538 return ERR_PTR(ret);
1539}
1540
1541static inline void save_comm(char *comm, struct task_struct *task)
1542{
1543 if (!task->pid) {
1544 strcpy(comm, "<idle>");
1545 return;
1546 }
1547
1548 if (WARN_ON_ONCE(task->pid < 0)) {
1549 strcpy(comm, "<XXX>");
1550 return;
1551 }
1552
1553 strncpy(comm, task->comm, TASK_COMM_LEN);
1554}
1555
1556static void hist_elt_data_free(struct hist_elt_data *elt_data)
1557{
1558 unsigned int i;
1559
1560 for (i = 0; i < elt_data->n_field_var_str; i++)
1561 kfree(elt_data->field_var_str[i]);
1562
1563 kfree(elt_data->field_var_str);
1564
1565 kfree(elt_data->comm);
1566 kfree(elt_data);
1567}
1568
1569static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1570{
1571 struct hist_elt_data *elt_data = elt->private_data;
1572
1573 hist_elt_data_free(elt_data);
1574}
1575
1576static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1577{
1578 struct hist_trigger_data *hist_data = elt->map->private_data;
1579 unsigned int size = TASK_COMM_LEN;
1580 struct hist_elt_data *elt_data;
1581 struct hist_field *hist_field;
1582 unsigned int i, n_str;
1583
1584 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1585 if (!elt_data)
1586 return -ENOMEM;
1587
1588 for_each_hist_field(i, hist_data) {
1589 hist_field = hist_data->fields[i];
1590
1591 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1592 elt_data->comm = kzalloc(size, GFP_KERNEL);
1593 if (!elt_data->comm) {
1594 kfree(elt_data);
1595 return -ENOMEM;
1596 }
1597 break;
1598 }
1599 }
1600
1601 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1602 hist_data->n_var_str;
1603 if (n_str > SYNTH_FIELDS_MAX) {
1604 hist_elt_data_free(elt_data);
1605 return -EINVAL;
1606 }
1607
1608 BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1609
1610 size = STR_VAR_LEN_MAX;
1611
1612 elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1613 if (!elt_data->field_var_str) {
1614 hist_elt_data_free(elt_data);
1615 return -EINVAL;
1616 }
1617 elt_data->n_field_var_str = n_str;
1618
1619 for (i = 0; i < n_str; i++) {
1620 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1621 if (!elt_data->field_var_str[i]) {
1622 hist_elt_data_free(elt_data);
1623 return -ENOMEM;
1624 }
1625 }
1626
1627 elt->private_data = elt_data;
1628
1629 return 0;
1630}
1631
1632static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1633{
1634 struct hist_elt_data *elt_data = elt->private_data;
1635
1636 if (elt_data->comm)
1637 save_comm(elt_data->comm, current);
1638}
1639
1640static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1641 .elt_alloc = hist_trigger_elt_data_alloc,
1642 .elt_free = hist_trigger_elt_data_free,
1643 .elt_init = hist_trigger_elt_data_init,
1644};
1645
1646static const char *get_hist_field_flags(struct hist_field *hist_field)
1647{
1648 const char *flags_str = NULL;
1649
1650 if (hist_field->flags & HIST_FIELD_FL_HEX)
1651 flags_str = "hex";
1652 else if (hist_field->flags & HIST_FIELD_FL_SYM)
1653 flags_str = "sym";
1654 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1655 flags_str = "sym-offset";
1656 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1657 flags_str = "execname";
1658 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1659 flags_str = "syscall";
1660 else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1661 flags_str = "log2";
1662 else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1663 flags_str = "buckets";
1664 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1665 flags_str = "usecs";
1666
1667 return flags_str;
1668}
1669
1670static void expr_field_str(struct hist_field *field, char *expr)
1671{
1672 if (field->flags & HIST_FIELD_FL_VAR_REF)
1673 strcat(expr, "$");
1674 else if (field->flags & HIST_FIELD_FL_CONST) {
1675 char str[HIST_CONST_DIGITS_MAX];
1676
1677 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1678 strcat(expr, str);
1679 }
1680
1681 strcat(expr, hist_field_name(field, 0));
1682
1683 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1684 const char *flags_str = get_hist_field_flags(field);
1685
1686 if (flags_str) {
1687 strcat(expr, ".");
1688 strcat(expr, flags_str);
1689 }
1690 }
1691}
1692
1693static char *expr_str(struct hist_field *field, unsigned int level)
1694{
1695 char *expr;
1696
1697 if (level > 1)
1698 return NULL;
1699
1700 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1701 if (!expr)
1702 return NULL;
1703
1704 if (!field->operands[0]) {
1705 expr_field_str(field, expr);
1706 return expr;
1707 }
1708
1709 if (field->operator == FIELD_OP_UNARY_MINUS) {
1710 char *subexpr;
1711
1712 strcat(expr, "-(");
1713 subexpr = expr_str(field->operands[0], ++level);
1714 if (!subexpr) {
1715 kfree(expr);
1716 return NULL;
1717 }
1718 strcat(expr, subexpr);
1719 strcat(expr, ")");
1720
1721 kfree(subexpr);
1722
1723 return expr;
1724 }
1725
1726 expr_field_str(field->operands[0], expr);
1727
1728 switch (field->operator) {
1729 case FIELD_OP_MINUS:
1730 strcat(expr, "-");
1731 break;
1732 case FIELD_OP_PLUS:
1733 strcat(expr, "+");
1734 break;
1735 case FIELD_OP_DIV:
1736 strcat(expr, "/");
1737 break;
1738 case FIELD_OP_MULT:
1739 strcat(expr, "*");
1740 break;
1741 default:
1742 kfree(expr);
1743 return NULL;
1744 }
1745
1746 expr_field_str(field->operands[1], expr);
1747
1748 return expr;
1749}
1750
1751
1752
1753
1754
1755static int contains_operator(char *str, char **sep)
1756{
1757 enum field_op_id field_op = FIELD_OP_NONE;
1758 char *minus_op, *plus_op, *div_op, *mult_op;
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775 minus_op = strrchr(str, '-');
1776 if (minus_op) {
1777
1778
1779
1780
1781 if (minus_op == str) {
1782 field_op = FIELD_OP_UNARY_MINUS;
1783 goto out;
1784 }
1785
1786 field_op = FIELD_OP_MINUS;
1787 }
1788
1789 plus_op = strrchr(str, '+');
1790 if (plus_op || minus_op) {
1791
1792
1793
1794
1795 if (plus_op > minus_op)
1796 field_op = FIELD_OP_PLUS;
1797 goto out;
1798 }
1799
1800
1801
1802
1803
1804 div_op = strrchr(str, '/');
1805 if (div_op)
1806 field_op = FIELD_OP_DIV;
1807
1808 mult_op = strrchr(str, '*');
1809
1810
1811
1812
1813 if (mult_op > div_op)
1814 field_op = FIELD_OP_MULT;
1815
1816out:
1817 if (sep) {
1818 switch (field_op) {
1819 case FIELD_OP_UNARY_MINUS:
1820 case FIELD_OP_MINUS:
1821 *sep = minus_op;
1822 break;
1823 case FIELD_OP_PLUS:
1824 *sep = plus_op;
1825 break;
1826 case FIELD_OP_DIV:
1827 *sep = div_op;
1828 break;
1829 case FIELD_OP_MULT:
1830 *sep = mult_op;
1831 break;
1832 case FIELD_OP_NONE:
1833 default:
1834 *sep = NULL;
1835 break;
1836 }
1837 }
1838
1839 return field_op;
1840}
1841
1842static void get_hist_field(struct hist_field *hist_field)
1843{
1844 hist_field->ref++;
1845}
1846
1847static void __destroy_hist_field(struct hist_field *hist_field)
1848{
1849 if (--hist_field->ref > 1)
1850 return;
1851
1852 kfree(hist_field->var.name);
1853 kfree(hist_field->name);
1854
1855
1856 kfree_const(hist_field->type);
1857
1858 kfree(hist_field->system);
1859 kfree(hist_field->event_name);
1860
1861 kfree(hist_field);
1862}
1863
1864static void destroy_hist_field(struct hist_field *hist_field,
1865 unsigned int level)
1866{
1867 unsigned int i;
1868
1869 if (level > 3)
1870 return;
1871
1872 if (!hist_field)
1873 return;
1874
1875 if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1876 return;
1877
1878 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1879 destroy_hist_field(hist_field->operands[i], level + 1);
1880
1881 __destroy_hist_field(hist_field);
1882}
1883
1884static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1885 struct ftrace_event_field *field,
1886 unsigned long flags,
1887 char *var_name)
1888{
1889 struct hist_field *hist_field;
1890
1891 if (field && is_function_field(field))
1892 return NULL;
1893
1894 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1895 if (!hist_field)
1896 return NULL;
1897
1898 hist_field->ref = 1;
1899
1900 hist_field->hist_data = hist_data;
1901
1902 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1903 goto out;
1904
1905 if (flags & HIST_FIELD_FL_VAR_REF) {
1906 hist_field->fn = hist_field_var_ref;
1907 goto out;
1908 }
1909
1910 if (flags & HIST_FIELD_FL_HITCOUNT) {
1911 hist_field->fn = hist_field_counter;
1912 hist_field->size = sizeof(u64);
1913 hist_field->type = "u64";
1914 goto out;
1915 }
1916
1917 if (flags & HIST_FIELD_FL_CONST) {
1918 hist_field->fn = hist_field_const;
1919 hist_field->size = sizeof(u64);
1920 hist_field->type = kstrdup("u64", GFP_KERNEL);
1921 if (!hist_field->type)
1922 goto free;
1923 goto out;
1924 }
1925
1926 if (flags & HIST_FIELD_FL_STACKTRACE) {
1927 hist_field->fn = hist_field_none;
1928 goto out;
1929 }
1930
1931 if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1932 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1933 hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
1934 hist_field_bucket;
1935 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1936 hist_field->size = hist_field->operands[0]->size;
1937 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1938 if (!hist_field->type)
1939 goto free;
1940 goto out;
1941 }
1942
1943 if (flags & HIST_FIELD_FL_TIMESTAMP) {
1944 hist_field->fn = hist_field_timestamp;
1945 hist_field->size = sizeof(u64);
1946 hist_field->type = "u64";
1947 goto out;
1948 }
1949
1950 if (flags & HIST_FIELD_FL_CPU) {
1951 hist_field->fn = hist_field_cpu;
1952 hist_field->size = sizeof(int);
1953 hist_field->type = "unsigned int";
1954 goto out;
1955 }
1956
1957 if (WARN_ON_ONCE(!field))
1958 goto out;
1959
1960
1961 if (is_string_field(field) &&
1962 (field->filter_type != FILTER_PTR_STRING)) {
1963 flags |= HIST_FIELD_FL_STRING;
1964
1965 hist_field->size = MAX_FILTER_STR_VAL;
1966 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1967 if (!hist_field->type)
1968 goto free;
1969
1970 if (field->filter_type == FILTER_STATIC_STRING) {
1971 hist_field->fn = hist_field_string;
1972 hist_field->size = field->size;
1973 } else if (field->filter_type == FILTER_DYN_STRING) {
1974 hist_field->fn = hist_field_dynstring;
1975 } else if (field->filter_type == FILTER_RDYN_STRING)
1976 hist_field->fn = hist_field_reldynstring;
1977 else
1978 hist_field->fn = hist_field_pstring;
1979 } else {
1980 hist_field->size = field->size;
1981 hist_field->is_signed = field->is_signed;
1982 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1983 if (!hist_field->type)
1984 goto free;
1985
1986 hist_field->fn = select_value_fn(field->size,
1987 field->is_signed);
1988 if (!hist_field->fn) {
1989 destroy_hist_field(hist_field, 0);
1990 return NULL;
1991 }
1992 }
1993 out:
1994 hist_field->field = field;
1995 hist_field->flags = flags;
1996
1997 if (var_name) {
1998 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1999 if (!hist_field->var.name)
2000 goto free;
2001 }
2002
2003 return hist_field;
2004 free:
2005 destroy_hist_field(hist_field, 0);
2006 return NULL;
2007}
2008
2009static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2010{
2011 unsigned int i;
2012
2013 for (i = 0; i < HIST_FIELDS_MAX; i++) {
2014 if (hist_data->fields[i]) {
2015 destroy_hist_field(hist_data->fields[i], 0);
2016 hist_data->fields[i] = NULL;
2017 }
2018 }
2019
2020 for (i = 0; i < hist_data->n_var_refs; i++) {
2021 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2022 __destroy_hist_field(hist_data->var_refs[i]);
2023 hist_data->var_refs[i] = NULL;
2024 }
2025}
2026
2027static int init_var_ref(struct hist_field *ref_field,
2028 struct hist_field *var_field,
2029 char *system, char *event_name)
2030{
2031 int err = 0;
2032
2033 ref_field->var.idx = var_field->var.idx;
2034 ref_field->var.hist_data = var_field->hist_data;
2035 ref_field->size = var_field->size;
2036 ref_field->is_signed = var_field->is_signed;
2037 ref_field->flags |= var_field->flags &
2038 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2039
2040 if (system) {
2041 ref_field->system = kstrdup(system, GFP_KERNEL);
2042 if (!ref_field->system)
2043 return -ENOMEM;
2044 }
2045
2046 if (event_name) {
2047 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2048 if (!ref_field->event_name) {
2049 err = -ENOMEM;
2050 goto free;
2051 }
2052 }
2053
2054 if (var_field->var.name) {
2055 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2056 if (!ref_field->name) {
2057 err = -ENOMEM;
2058 goto free;
2059 }
2060 } else if (var_field->name) {
2061 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2062 if (!ref_field->name) {
2063 err = -ENOMEM;
2064 goto free;
2065 }
2066 }
2067
2068 ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2069 if (!ref_field->type) {
2070 err = -ENOMEM;
2071 goto free;
2072 }
2073 out:
2074 return err;
2075 free:
2076 kfree(ref_field->system);
2077 kfree(ref_field->event_name);
2078 kfree(ref_field->name);
2079
2080 goto out;
2081}
2082
2083static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2084 struct hist_field *var_field)
2085{
2086 struct hist_field *ref_field;
2087 int i;
2088
2089 for (i = 0; i < hist_data->n_var_refs; i++) {
2090 ref_field = hist_data->var_refs[i];
2091 if (ref_field->var.idx == var_field->var.idx &&
2092 ref_field->var.hist_data == var_field->hist_data)
2093 return i;
2094 }
2095
2096 return -ENOENT;
2097}
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2115 struct hist_field *var_field,
2116 char *system, char *event_name)
2117{
2118 unsigned long flags = HIST_FIELD_FL_VAR_REF;
2119 struct hist_field *ref_field;
2120 int i;
2121
2122
2123 for (i = 0; i < hist_data->n_var_refs; i++) {
2124 ref_field = hist_data->var_refs[i];
2125 if (ref_field->var.idx == var_field->var.idx &&
2126 ref_field->var.hist_data == var_field->hist_data) {
2127 get_hist_field(ref_field);
2128 return ref_field;
2129 }
2130 }
2131
2132 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2133 if (ref_field) {
2134 if (init_var_ref(ref_field, var_field, system, event_name)) {
2135 destroy_hist_field(ref_field, 0);
2136 return NULL;
2137 }
2138
2139 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2140 ref_field->var_ref_idx = hist_data->n_var_refs++;
2141 }
2142
2143 return ref_field;
2144}
2145
2146static bool is_var_ref(char *var_name)
2147{
2148 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2149 return false;
2150
2151 return true;
2152}
2153
2154static char *field_name_from_var(struct hist_trigger_data *hist_data,
2155 char *var_name)
2156{
2157 char *name, *field;
2158 unsigned int i;
2159
2160 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2161 name = hist_data->attrs->var_defs.name[i];
2162
2163 if (strcmp(var_name, name) == 0) {
2164 field = hist_data->attrs->var_defs.expr[i];
2165 if (contains_operator(field, NULL) || is_var_ref(field))
2166 continue;
2167 return field;
2168 }
2169 }
2170
2171 return NULL;
2172}
2173
2174static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2175 char *system, char *event_name,
2176 char *var_name)
2177{
2178 struct trace_event_call *call;
2179
2180 if (system && event_name) {
2181 call = hist_data->event_file->event_call;
2182
2183 if (strcmp(system, call->class->system) != 0)
2184 return NULL;
2185
2186 if (strcmp(event_name, trace_event_name(call)) != 0)
2187 return NULL;
2188 }
2189
2190 if (!!system != !!event_name)
2191 return NULL;
2192
2193 if (!is_var_ref(var_name))
2194 return NULL;
2195
2196 var_name++;
2197
2198 return field_name_from_var(hist_data, var_name);
2199}
2200
2201static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2202 char *system, char *event_name,
2203 char *var_name)
2204{
2205 struct hist_field *var_field = NULL, *ref_field = NULL;
2206 struct trace_array *tr = hist_data->event_file->tr;
2207
2208 if (!is_var_ref(var_name))
2209 return NULL;
2210
2211 var_name++;
2212
2213 var_field = find_event_var(hist_data, system, event_name, var_name);
2214 if (var_field)
2215 ref_field = create_var_ref(hist_data, var_field,
2216 system, event_name);
2217
2218 if (!ref_field)
2219 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2220
2221 return ref_field;
2222}
2223
2224static struct ftrace_event_field *
2225parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2226 char *field_str, unsigned long *flags, unsigned long *buckets)
2227{
2228 struct ftrace_event_field *field = NULL;
2229 char *field_name, *modifier, *str;
2230 struct trace_array *tr = file->tr;
2231
2232 modifier = str = kstrdup(field_str, GFP_KERNEL);
2233 if (!modifier)
2234 return ERR_PTR(-ENOMEM);
2235
2236 field_name = strsep(&modifier, ".");
2237 if (modifier) {
2238 if (strcmp(modifier, "hex") == 0)
2239 *flags |= HIST_FIELD_FL_HEX;
2240 else if (strcmp(modifier, "sym") == 0)
2241 *flags |= HIST_FIELD_FL_SYM;
2242
2243
2244
2245
2246 else if (strcmp(modifier, "symXoffset") == 0)
2247 *flags |= HIST_FIELD_FL_SYM_OFFSET;
2248 else if ((strcmp(modifier, "execname") == 0) &&
2249 (strcmp(field_name, "common_pid") == 0))
2250 *flags |= HIST_FIELD_FL_EXECNAME;
2251 else if (strcmp(modifier, "syscall") == 0)
2252 *flags |= HIST_FIELD_FL_SYSCALL;
2253 else if (strcmp(modifier, "log2") == 0)
2254 *flags |= HIST_FIELD_FL_LOG2;
2255 else if (strcmp(modifier, "usecs") == 0)
2256 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2257 else if (strncmp(modifier, "bucket", 6) == 0) {
2258 int ret;
2259
2260 modifier += 6;
2261
2262 if (*modifier == 's')
2263 modifier++;
2264 if (*modifier != '=')
2265 goto error;
2266 modifier++;
2267 ret = kstrtoul(modifier, 0, buckets);
2268 if (ret || !(*buckets))
2269 goto error;
2270 *flags |= HIST_FIELD_FL_BUCKET;
2271 } else {
2272 error:
2273 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2274 field = ERR_PTR(-EINVAL);
2275 goto out;
2276 }
2277 }
2278
2279 if (strcmp(field_name, "common_timestamp") == 0) {
2280 *flags |= HIST_FIELD_FL_TIMESTAMP;
2281 hist_data->enable_timestamps = true;
2282 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2283 hist_data->attrs->ts_in_usecs = true;
2284 } else if (strcmp(field_name, "common_cpu") == 0)
2285 *flags |= HIST_FIELD_FL_CPU;
2286 else {
2287 field = trace_find_event_field(file->event_call, field_name);
2288 if (!field || !field->size) {
2289
2290
2291
2292
2293
2294 if (field && field->filter_type == FILTER_CPU) {
2295 *flags |= HIST_FIELD_FL_CPU;
2296 } else {
2297 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2298 errpos(field_name));
2299 field = ERR_PTR(-EINVAL);
2300 goto out;
2301 }
2302 }
2303 }
2304 out:
2305 kfree(str);
2306
2307 return field;
2308}
2309
2310static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2311 struct hist_field *var_ref,
2312 char *var_name)
2313{
2314 struct hist_field *alias = NULL;
2315 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2316
2317 alias = create_hist_field(hist_data, NULL, flags, var_name);
2318 if (!alias)
2319 return NULL;
2320
2321 alias->fn = var_ref->fn;
2322 alias->operands[0] = var_ref;
2323
2324 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2325 destroy_hist_field(alias, 0);
2326 return NULL;
2327 }
2328
2329 alias->var_ref_idx = var_ref->var_ref_idx;
2330
2331 return alias;
2332}
2333
2334static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2335 char *str, char *var_name,
2336 unsigned long *flags)
2337{
2338 struct trace_array *tr = hist_data->event_file->tr;
2339 struct hist_field *field = NULL;
2340 u64 constant;
2341
2342 if (kstrtoull(str, 0, &constant)) {
2343 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2344 return NULL;
2345 }
2346
2347 *flags |= HIST_FIELD_FL_CONST;
2348 field = create_hist_field(hist_data, NULL, *flags, var_name);
2349 if (!field)
2350 return NULL;
2351
2352 field->constant = constant;
2353
2354 return field;
2355}
2356
2357static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2358 struct trace_event_file *file, char *str,
2359 unsigned long *flags, char *var_name)
2360{
2361 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2362 struct ftrace_event_field *field = NULL;
2363 struct hist_field *hist_field = NULL;
2364 unsigned long buckets = 0;
2365 int ret = 0;
2366
2367 if (isdigit(str[0])) {
2368 hist_field = parse_const(hist_data, str, var_name, flags);
2369 if (!hist_field) {
2370 ret = -EINVAL;
2371 goto out;
2372 }
2373 return hist_field;
2374 }
2375
2376 s = strchr(str, '.');
2377 if (s) {
2378 s = strchr(++s, '.');
2379 if (s) {
2380 ref_system = strsep(&str, ".");
2381 if (!str) {
2382 ret = -EINVAL;
2383 goto out;
2384 }
2385 ref_event = strsep(&str, ".");
2386 if (!str) {
2387 ret = -EINVAL;
2388 goto out;
2389 }
2390 ref_var = str;
2391 }
2392 }
2393
2394 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2395 if (!s) {
2396 hist_field = parse_var_ref(hist_data, ref_system,
2397 ref_event, ref_var);
2398 if (hist_field) {
2399 if (var_name) {
2400 hist_field = create_alias(hist_data, hist_field, var_name);
2401 if (!hist_field) {
2402 ret = -ENOMEM;
2403 goto out;
2404 }
2405 }
2406 return hist_field;
2407 }
2408 } else
2409 str = s;
2410
2411 field = parse_field(hist_data, file, str, flags, &buckets);
2412 if (IS_ERR(field)) {
2413 ret = PTR_ERR(field);
2414 goto out;
2415 }
2416
2417 hist_field = create_hist_field(hist_data, field, *flags, var_name);
2418 if (!hist_field) {
2419 ret = -ENOMEM;
2420 goto out;
2421 }
2422 hist_field->buckets = buckets;
2423
2424 return hist_field;
2425 out:
2426 return ERR_PTR(ret);
2427}
2428
2429static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2430 struct trace_event_file *file,
2431 char *str, unsigned long flags,
2432 char *var_name, unsigned int *n_subexprs);
2433
2434static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2435 struct trace_event_file *file,
2436 char *str, unsigned long flags,
2437 char *var_name, unsigned int *n_subexprs)
2438{
2439 struct hist_field *operand1, *expr = NULL;
2440 unsigned long operand_flags;
2441 int ret = 0;
2442 char *s;
2443
2444
2445 ++*n_subexprs;
2446
2447
2448
2449 if (*n_subexprs > 3) {
2450 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2451 ret = -EINVAL;
2452 goto free;
2453 }
2454
2455 str++;
2456
2457 s = strchr(str, '(');
2458 if (s)
2459 str++;
2460 else {
2461 ret = -EINVAL;
2462 goto free;
2463 }
2464
2465 s = strrchr(str, ')');
2466 if (s) {
2467
2468 if (*(s+1) != '\0') {
2469 hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2470 errpos(str));
2471 ret = -EINVAL;
2472 goto free;
2473 }
2474 *s = '\0';
2475 }
2476 else {
2477 ret = -EINVAL;
2478 goto free;
2479 }
2480
2481 flags |= HIST_FIELD_FL_EXPR;
2482 expr = create_hist_field(hist_data, NULL, flags, var_name);
2483 if (!expr) {
2484 ret = -ENOMEM;
2485 goto free;
2486 }
2487
2488 operand_flags = 0;
2489 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2490 if (IS_ERR(operand1)) {
2491 ret = PTR_ERR(operand1);
2492 goto free;
2493 }
2494 if (operand1->flags & HIST_FIELD_FL_STRING) {
2495
2496 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2497 destroy_hist_field(operand1, 0);
2498 ret = -EINVAL;
2499 goto free;
2500 }
2501
2502 expr->flags |= operand1->flags &
2503 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2504 expr->fn = hist_field_unary_minus;
2505 expr->operands[0] = operand1;
2506 expr->size = operand1->size;
2507 expr->is_signed = operand1->is_signed;
2508 expr->operator = FIELD_OP_UNARY_MINUS;
2509 expr->name = expr_str(expr, 0);
2510 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2511 if (!expr->type) {
2512 ret = -ENOMEM;
2513 goto free;
2514 }
2515
2516 return expr;
2517 free:
2518 destroy_hist_field(expr, 0);
2519 return ERR_PTR(ret);
2520}
2521
2522
2523
2524
2525
2526static int check_expr_operands(struct trace_array *tr,
2527 struct hist_field *operand1,
2528 struct hist_field *operand2,
2529 struct hist_field **var1,
2530 struct hist_field **var2)
2531{
2532 unsigned long operand1_flags = operand1->flags;
2533 unsigned long operand2_flags = operand2->flags;
2534
2535 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2536 (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2537 struct hist_field *var;
2538
2539 var = find_var_field(operand1->var.hist_data, operand1->name);
2540 if (!var)
2541 return -EINVAL;
2542 operand1_flags = var->flags;
2543 *var1 = var;
2544 }
2545
2546 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2547 (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2548 struct hist_field *var;
2549
2550 var = find_var_field(operand2->var.hist_data, operand2->name);
2551 if (!var)
2552 return -EINVAL;
2553 operand2_flags = var->flags;
2554 *var2 = var;
2555 }
2556
2557 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2558 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2559 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2560 return -EINVAL;
2561 }
2562
2563 return 0;
2564}
2565
2566static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2567 struct trace_event_file *file,
2568 char *str, unsigned long flags,
2569 char *var_name, unsigned int *n_subexprs)
2570{
2571 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2572 struct hist_field *var1 = NULL, *var2 = NULL;
2573 unsigned long operand_flags, operand2_flags;
2574 int field_op, ret = -EINVAL;
2575 char *sep, *operand1_str;
2576 hist_field_fn_t op_fn;
2577 bool combine_consts;
2578
2579 if (*n_subexprs > 3) {
2580 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2581 return ERR_PTR(-EINVAL);
2582 }
2583
2584 field_op = contains_operator(str, &sep);
2585
2586 if (field_op == FIELD_OP_NONE)
2587 return parse_atom(hist_data, file, str, &flags, var_name);
2588
2589 if (field_op == FIELD_OP_UNARY_MINUS)
2590 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2591
2592
2593 ++*n_subexprs;
2594
2595
2596 if (!sep)
2597 return ERR_PTR(-EINVAL);
2598
2599 *sep = '\0';
2600 operand1_str = str;
2601 str = sep+1;
2602
2603
2604 if (*operand1_str == '\0' || *str == '\0')
2605 return ERR_PTR(-EINVAL);
2606
2607 operand_flags = 0;
2608
2609
2610 operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2611 if (IS_ERR(operand1))
2612 return ERR_CAST(operand1);
2613
2614 if (operand1->flags & HIST_FIELD_FL_STRING) {
2615 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2616 ret = -EINVAL;
2617 goto free_op1;
2618 }
2619
2620
2621 operand_flags = 0;
2622 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2623 if (IS_ERR(operand2)) {
2624 ret = PTR_ERR(operand2);
2625 goto free_op1;
2626 }
2627 if (operand2->flags & HIST_FIELD_FL_STRING) {
2628 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2629 ret = -EINVAL;
2630 goto free_operands;
2631 }
2632
2633 switch (field_op) {
2634 case FIELD_OP_MINUS:
2635 op_fn = hist_field_minus;
2636 break;
2637 case FIELD_OP_PLUS:
2638 op_fn = hist_field_plus;
2639 break;
2640 case FIELD_OP_DIV:
2641 op_fn = hist_field_div;
2642 break;
2643 case FIELD_OP_MULT:
2644 op_fn = hist_field_mult;
2645 break;
2646 default:
2647 ret = -EINVAL;
2648 goto free_operands;
2649 }
2650
2651 ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2652 if (ret)
2653 goto free_operands;
2654
2655 operand_flags = var1 ? var1->flags : operand1->flags;
2656 operand2_flags = var2 ? var2->flags : operand2->flags;
2657
2658
2659
2660
2661
2662 combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2663
2664 flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2665
2666 flags |= operand1->flags &
2667 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2668
2669 expr = create_hist_field(hist_data, NULL, flags, var_name);
2670 if (!expr) {
2671 ret = -ENOMEM;
2672 goto free_operands;
2673 }
2674
2675 operand1->read_once = true;
2676 operand2->read_once = true;
2677
2678
2679 expr->operands[0] = operand1;
2680 expr->operands[1] = operand2;
2681
2682 if (field_op == FIELD_OP_DIV &&
2683 operand2_flags & HIST_FIELD_FL_CONST) {
2684 u64 divisor = var2 ? var2->constant : operand2->constant;
2685
2686 if (!divisor) {
2687 hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2688 ret = -EDOM;
2689 goto free_expr;
2690 }
2691
2692
2693
2694
2695
2696 operand2->constant = divisor;
2697 op_fn = hist_field_get_div_fn(operand2);
2698 }
2699
2700 if (combine_consts) {
2701 if (var1)
2702 expr->operands[0] = var1;
2703 if (var2)
2704 expr->operands[1] = var2;
2705
2706 expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
2707
2708 expr->operands[0] = NULL;
2709 expr->operands[1] = NULL;
2710
2711
2712
2713
2714
2715 destroy_hist_field(operand2, 0);
2716 destroy_hist_field(operand1, 0);
2717
2718 expr->name = expr_str(expr, 0);
2719 } else {
2720 expr->fn = op_fn;
2721
2722
2723 expr->size = operand1->size;
2724 expr->is_signed = operand1->is_signed;
2725
2726 expr->operator = field_op;
2727 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2728 if (!expr->type) {
2729 ret = -ENOMEM;
2730 goto free_expr;
2731 }
2732
2733 expr->name = expr_str(expr, 0);
2734 }
2735
2736 return expr;
2737
2738free_operands:
2739 destroy_hist_field(operand2, 0);
2740free_op1:
2741 destroy_hist_field(operand1, 0);
2742 return ERR_PTR(ret);
2743
2744free_expr:
2745 destroy_hist_field(expr, 0);
2746 return ERR_PTR(ret);
2747}
2748
2749static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2750 struct trace_event_file *file)
2751{
2752 struct event_trigger_data *test;
2753
2754 lockdep_assert_held(&event_mutex);
2755
2756 list_for_each_entry(test, &file->triggers, list) {
2757 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2758 if (test->private_data == hist_data)
2759 return test->filter_str;
2760 }
2761 }
2762
2763 return NULL;
2764}
2765
2766static struct event_command trigger_hist_cmd;
2767static int event_hist_trigger_parse(struct event_command *cmd_ops,
2768 struct trace_event_file *file,
2769 char *glob, char *cmd, char *param);
2770
2771static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2772 struct hist_trigger_data *hist_data,
2773 unsigned int n_keys)
2774{
2775 struct hist_field *target_hist_field, *hist_field;
2776 unsigned int n, i, j;
2777
2778 if (hist_data->n_fields - hist_data->n_vals != n_keys)
2779 return false;
2780
2781 i = hist_data->n_vals;
2782 j = target_hist_data->n_vals;
2783
2784 for (n = 0; n < n_keys; n++) {
2785 hist_field = hist_data->fields[i + n];
2786 target_hist_field = target_hist_data->fields[j + n];
2787
2788 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2789 return false;
2790 if (hist_field->size != target_hist_field->size)
2791 return false;
2792 if (hist_field->is_signed != target_hist_field->is_signed)
2793 return false;
2794 }
2795
2796 return true;
2797}
2798
2799static struct hist_trigger_data *
2800find_compatible_hist(struct hist_trigger_data *target_hist_data,
2801 struct trace_event_file *file)
2802{
2803 struct hist_trigger_data *hist_data;
2804 struct event_trigger_data *test;
2805 unsigned int n_keys;
2806
2807 lockdep_assert_held(&event_mutex);
2808
2809 n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2810
2811 list_for_each_entry(test, &file->triggers, list) {
2812 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2813 hist_data = test->private_data;
2814
2815 if (compatible_keys(target_hist_data, hist_data, n_keys))
2816 return hist_data;
2817 }
2818 }
2819
2820 return NULL;
2821}
2822
2823static struct trace_event_file *event_file(struct trace_array *tr,
2824 char *system, char *event_name)
2825{
2826 struct trace_event_file *file;
2827
2828 file = __find_event_file(tr, system, event_name);
2829 if (!file)
2830 return ERR_PTR(-EINVAL);
2831
2832 return file;
2833}
2834
2835static struct hist_field *
2836find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2837 char *system, char *event_name, char *field_name)
2838{
2839 struct hist_field *event_var;
2840 char *synthetic_name;
2841
2842 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2843 if (!synthetic_name)
2844 return ERR_PTR(-ENOMEM);
2845
2846 strcpy(synthetic_name, "synthetic_");
2847 strcat(synthetic_name, field_name);
2848
2849 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2850
2851 kfree(synthetic_name);
2852
2853 return event_var;
2854}
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881static struct hist_field *
2882create_field_var_hist(struct hist_trigger_data *target_hist_data,
2883 char *subsys_name, char *event_name, char *field_name)
2884{
2885 struct trace_array *tr = target_hist_data->event_file->tr;
2886 struct hist_trigger_data *hist_data;
2887 unsigned int i, n, first = true;
2888 struct field_var_hist *var_hist;
2889 struct trace_event_file *file;
2890 struct hist_field *key_field;
2891 struct hist_field *event_var;
2892 char *saved_filter;
2893 char *cmd;
2894 int ret;
2895
2896 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2897 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2898 return ERR_PTR(-EINVAL);
2899 }
2900
2901 file = event_file(tr, subsys_name, event_name);
2902
2903 if (IS_ERR(file)) {
2904 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2905 ret = PTR_ERR(file);
2906 return ERR_PTR(ret);
2907 }
2908
2909
2910
2911
2912
2913
2914
2915 hist_data = find_compatible_hist(target_hist_data, file);
2916 if (!hist_data) {
2917 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2918 return ERR_PTR(-EINVAL);
2919 }
2920
2921
2922 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2923 event_name, field_name);
2924 if (!IS_ERR_OR_NULL(event_var))
2925 return event_var;
2926
2927 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2928 if (!var_hist)
2929 return ERR_PTR(-ENOMEM);
2930
2931 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2932 if (!cmd) {
2933 kfree(var_hist);
2934 return ERR_PTR(-ENOMEM);
2935 }
2936
2937
2938 strcat(cmd, "keys=");
2939
2940 for_each_hist_key_field(i, hist_data) {
2941 key_field = hist_data->fields[i];
2942 if (!first)
2943 strcat(cmd, ",");
2944 strcat(cmd, key_field->field->name);
2945 first = false;
2946 }
2947
2948
2949 strcat(cmd, ":synthetic_");
2950 strcat(cmd, field_name);
2951 strcat(cmd, "=");
2952 strcat(cmd, field_name);
2953
2954
2955 saved_filter = find_trigger_filter(hist_data, file);
2956 if (saved_filter) {
2957 strcat(cmd, " if ");
2958 strcat(cmd, saved_filter);
2959 }
2960
2961 var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2962 if (!var_hist->cmd) {
2963 kfree(cmd);
2964 kfree(var_hist);
2965 return ERR_PTR(-ENOMEM);
2966 }
2967
2968
2969 var_hist->hist_data = hist_data;
2970
2971
2972 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
2973 "", "hist", cmd);
2974 if (ret) {
2975 kfree(cmd);
2976 kfree(var_hist->cmd);
2977 kfree(var_hist);
2978 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2979 return ERR_PTR(ret);
2980 }
2981
2982 kfree(cmd);
2983
2984
2985 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2986 event_name, field_name);
2987 if (IS_ERR_OR_NULL(event_var)) {
2988 kfree(var_hist->cmd);
2989 kfree(var_hist);
2990 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2991 return ERR_PTR(-EINVAL);
2992 }
2993
2994 n = target_hist_data->n_field_var_hists;
2995 target_hist_data->field_var_hists[n] = var_hist;
2996 target_hist_data->n_field_var_hists++;
2997
2998 return event_var;
2999}
3000
3001static struct hist_field *
3002find_target_event_var(struct hist_trigger_data *hist_data,
3003 char *subsys_name, char *event_name, char *var_name)
3004{
3005 struct trace_event_file *file = hist_data->event_file;
3006 struct hist_field *hist_field = NULL;
3007
3008 if (subsys_name) {
3009 struct trace_event_call *call;
3010
3011 if (!event_name)
3012 return NULL;
3013
3014 call = file->event_call;
3015
3016 if (strcmp(subsys_name, call->class->system) != 0)
3017 return NULL;
3018
3019 if (strcmp(event_name, trace_event_name(call)) != 0)
3020 return NULL;
3021 }
3022
3023 hist_field = find_var_field(hist_data, var_name);
3024
3025 return hist_field;
3026}
3027
3028static inline void __update_field_vars(struct tracing_map_elt *elt,
3029 struct trace_buffer *buffer,
3030 struct ring_buffer_event *rbe,
3031 void *rec,
3032 struct field_var **field_vars,
3033 unsigned int n_field_vars,
3034 unsigned int field_var_str_start)
3035{
3036 struct hist_elt_data *elt_data = elt->private_data;
3037 unsigned int i, j, var_idx;
3038 u64 var_val;
3039
3040 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3041 struct field_var *field_var = field_vars[i];
3042 struct hist_field *var = field_var->var;
3043 struct hist_field *val = field_var->val;
3044
3045 var_val = val->fn(val, elt, buffer, rbe, rec);
3046 var_idx = var->var.idx;
3047
3048 if (val->flags & HIST_FIELD_FL_STRING) {
3049 char *str = elt_data->field_var_str[j++];
3050 char *val_str = (char *)(uintptr_t)var_val;
3051 unsigned int size;
3052
3053 size = min(val->size, STR_VAR_LEN_MAX);
3054 strscpy(str, val_str, size);
3055 var_val = (u64)(uintptr_t)str;
3056 }
3057 tracing_map_set_var(elt, var_idx, var_val);
3058 }
3059}
3060
3061static void update_field_vars(struct hist_trigger_data *hist_data,
3062 struct tracing_map_elt *elt,
3063 struct trace_buffer *buffer,
3064 struct ring_buffer_event *rbe,
3065 void *rec)
3066{
3067 __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3068 hist_data->n_field_vars, 0);
3069}
3070
3071static void save_track_data_vars(struct hist_trigger_data *hist_data,
3072 struct tracing_map_elt *elt,
3073 struct trace_buffer *buffer, void *rec,
3074 struct ring_buffer_event *rbe, void *key,
3075 struct action_data *data, u64 *var_ref_vals)
3076{
3077 __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3078 hist_data->n_save_vars, hist_data->n_field_var_str);
3079}
3080
3081static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3082 struct trace_event_file *file,
3083 char *name, int size, const char *type)
3084{
3085 struct hist_field *var;
3086 int idx;
3087
3088 if (find_var(hist_data, file, name) && !hist_data->remove) {
3089 var = ERR_PTR(-EINVAL);
3090 goto out;
3091 }
3092
3093 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3094 if (!var) {
3095 var = ERR_PTR(-ENOMEM);
3096 goto out;
3097 }
3098
3099 idx = tracing_map_add_var(hist_data->map);
3100 if (idx < 0) {
3101 kfree(var);
3102 var = ERR_PTR(-EINVAL);
3103 goto out;
3104 }
3105
3106 var->ref = 1;
3107 var->flags = HIST_FIELD_FL_VAR;
3108 var->var.idx = idx;
3109 var->var.hist_data = var->hist_data = hist_data;
3110 var->size = size;
3111 var->var.name = kstrdup(name, GFP_KERNEL);
3112 var->type = kstrdup_const(type, GFP_KERNEL);
3113 if (!var->var.name || !var->type) {
3114 kfree_const(var->type);
3115 kfree(var->var.name);
3116 kfree(var);
3117 var = ERR_PTR(-ENOMEM);
3118 }
3119 out:
3120 return var;
3121}
3122
3123static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3124 struct trace_event_file *file,
3125 char *field_name)
3126{
3127 struct hist_field *val = NULL, *var = NULL;
3128 unsigned long flags = HIST_FIELD_FL_VAR;
3129 struct trace_array *tr = file->tr;
3130 struct field_var *field_var;
3131 int ret = 0;
3132
3133 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3134 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3135 ret = -EINVAL;
3136 goto err;
3137 }
3138
3139 val = parse_atom(hist_data, file, field_name, &flags, NULL);
3140 if (IS_ERR(val)) {
3141 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3142 ret = PTR_ERR(val);
3143 goto err;
3144 }
3145
3146 var = create_var(hist_data, file, field_name, val->size, val->type);
3147 if (IS_ERR(var)) {
3148 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3149 kfree(val);
3150 ret = PTR_ERR(var);
3151 goto err;
3152 }
3153
3154 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3155 if (!field_var) {
3156 kfree(val);
3157 kfree(var);
3158 ret = -ENOMEM;
3159 goto err;
3160 }
3161
3162 field_var->var = var;
3163 field_var->val = val;
3164 out:
3165 return field_var;
3166 err:
3167 field_var = ERR_PTR(ret);
3168 goto out;
3169}
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191static struct field_var *
3192create_target_field_var(struct hist_trigger_data *target_hist_data,
3193 char *subsys_name, char *event_name, char *var_name)
3194{
3195 struct trace_event_file *file = target_hist_data->event_file;
3196
3197 if (subsys_name) {
3198 struct trace_event_call *call;
3199
3200 if (!event_name)
3201 return NULL;
3202
3203 call = file->event_call;
3204
3205 if (strcmp(subsys_name, call->class->system) != 0)
3206 return NULL;
3207
3208 if (strcmp(event_name, trace_event_name(call)) != 0)
3209 return NULL;
3210 }
3211
3212 return create_field_var(target_hist_data, file, var_name);
3213}
3214
3215static bool check_track_val_max(u64 track_val, u64 var_val)
3216{
3217 if (var_val <= track_val)
3218 return false;
3219
3220 return true;
3221}
3222
3223static bool check_track_val_changed(u64 track_val, u64 var_val)
3224{
3225 if (var_val == track_val)
3226 return false;
3227
3228 return true;
3229}
3230
3231static u64 get_track_val(struct hist_trigger_data *hist_data,
3232 struct tracing_map_elt *elt,
3233 struct action_data *data)
3234{
3235 unsigned int track_var_idx = data->track_data.track_var->var.idx;
3236 u64 track_val;
3237
3238 track_val = tracing_map_read_var(elt, track_var_idx);
3239
3240 return track_val;
3241}
3242
3243static void save_track_val(struct hist_trigger_data *hist_data,
3244 struct tracing_map_elt *elt,
3245 struct action_data *data, u64 var_val)
3246{
3247 unsigned int track_var_idx = data->track_data.track_var->var.idx;
3248
3249 tracing_map_set_var(elt, track_var_idx, var_val);
3250}
3251
3252static void save_track_data(struct hist_trigger_data *hist_data,
3253 struct tracing_map_elt *elt,
3254 struct trace_buffer *buffer, void *rec,
3255 struct ring_buffer_event *rbe, void *key,
3256 struct action_data *data, u64 *var_ref_vals)
3257{
3258 if (data->track_data.save_data)
3259 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3260 key, data, var_ref_vals);
3261}
3262
3263static bool check_track_val(struct tracing_map_elt *elt,
3264 struct action_data *data,
3265 u64 var_val)
3266{
3267 struct hist_trigger_data *hist_data;
3268 u64 track_val;
3269
3270 hist_data = data->track_data.track_var->hist_data;
3271 track_val = get_track_val(hist_data, elt, data);
3272
3273 return data->track_data.check_val(track_val, var_val);
3274}
3275
3276#ifdef CONFIG_TRACER_SNAPSHOT
3277static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3278{
3279
3280 struct track_data *track_data = tr->cond_snapshot->cond_data;
3281 struct hist_elt_data *elt_data, *track_elt_data;
3282 struct snapshot_context *context = cond_data;
3283 struct action_data *action;
3284 u64 track_val;
3285
3286 if (!track_data)
3287 return false;
3288
3289 action = track_data->action_data;
3290
3291 track_val = get_track_val(track_data->hist_data, context->elt,
3292 track_data->action_data);
3293
3294 if (!action->track_data.check_val(track_data->track_val, track_val))
3295 return false;
3296
3297 track_data->track_val = track_val;
3298 memcpy(track_data->key, context->key, track_data->key_len);
3299
3300 elt_data = context->elt->private_data;
3301 track_elt_data = track_data->elt.private_data;
3302 if (elt_data->comm)
3303 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3304
3305 track_data->updated = true;
3306
3307 return true;
3308}
3309
3310static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3311 struct tracing_map_elt *elt,
3312 struct trace_buffer *buffer, void *rec,
3313 struct ring_buffer_event *rbe, void *key,
3314 struct action_data *data,
3315 u64 *var_ref_vals)
3316{
3317 struct trace_event_file *file = hist_data->event_file;
3318 struct snapshot_context context;
3319
3320 context.elt = elt;
3321 context.key = key;
3322
3323 tracing_snapshot_cond(file->tr, &context);
3324}
3325
3326static void hist_trigger_print_key(struct seq_file *m,
3327 struct hist_trigger_data *hist_data,
3328 void *key,
3329 struct tracing_map_elt *elt);
3330
3331static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3332{
3333 unsigned int i;
3334
3335 if (!hist_data->n_actions)
3336 return NULL;
3337
3338 for (i = 0; i < hist_data->n_actions; i++) {
3339 struct action_data *data = hist_data->actions[i];
3340
3341 if (data->action == ACTION_SNAPSHOT)
3342 return data;
3343 }
3344
3345 return NULL;
3346}
3347
3348static void track_data_snapshot_print(struct seq_file *m,
3349 struct hist_trigger_data *hist_data)
3350{
3351 struct trace_event_file *file = hist_data->event_file;
3352 struct track_data *track_data;
3353 struct action_data *action;
3354
3355 track_data = tracing_cond_snapshot_data(file->tr);
3356 if (!track_data)
3357 return;
3358
3359 if (!track_data->updated)
3360 return;
3361
3362 action = snapshot_action(hist_data);
3363 if (!action)
3364 return;
3365
3366 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n");
3367 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3368 action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3369 action->track_data.var_str, track_data->track_val);
3370
3371 seq_puts(m, "\ttriggered by event with key: ");
3372 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3373 seq_putc(m, '\n');
3374}
3375#else
3376static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3377{
3378 return false;
3379}
3380static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3381 struct tracing_map_elt *elt,
3382 struct trace_buffer *buffer, void *rec,
3383 struct ring_buffer_event *rbe, void *key,
3384 struct action_data *data,
3385 u64 *var_ref_vals) {}
3386static void track_data_snapshot_print(struct seq_file *m,
3387 struct hist_trigger_data *hist_data) {}
3388#endif
3389
3390static void track_data_print(struct seq_file *m,
3391 struct hist_trigger_data *hist_data,
3392 struct tracing_map_elt *elt,
3393 struct action_data *data)
3394{
3395 u64 track_val = get_track_val(hist_data, elt, data);
3396 unsigned int i, save_var_idx;
3397
3398 if (data->handler == HANDLER_ONMAX)
3399 seq_printf(m, "\n\tmax: %10llu", track_val);
3400 else if (data->handler == HANDLER_ONCHANGE)
3401 seq_printf(m, "\n\tchanged: %10llu", track_val);
3402
3403 if (data->action == ACTION_SNAPSHOT)
3404 return;
3405
3406 for (i = 0; i < hist_data->n_save_vars; i++) {
3407 struct hist_field *save_val = hist_data->save_vars[i]->val;
3408 struct hist_field *save_var = hist_data->save_vars[i]->var;
3409 u64 val;
3410
3411 save_var_idx = save_var->var.idx;
3412
3413 val = tracing_map_read_var(elt, save_var_idx);
3414
3415 if (save_val->flags & HIST_FIELD_FL_STRING) {
3416 seq_printf(m, " %s: %-32s", save_var->var.name,
3417 (char *)(uintptr_t)(val));
3418 } else
3419 seq_printf(m, " %s: %10llu", save_var->var.name, val);
3420 }
3421}
3422
3423static void ontrack_action(struct hist_trigger_data *hist_data,
3424 struct tracing_map_elt *elt,
3425 struct trace_buffer *buffer, void *rec,
3426 struct ring_buffer_event *rbe, void *key,
3427 struct action_data *data, u64 *var_ref_vals)
3428{
3429 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3430
3431 if (check_track_val(elt, data, var_val)) {
3432 save_track_val(hist_data, elt, data, var_val);
3433 save_track_data(hist_data, elt, buffer, rec, rbe,
3434 key, data, var_ref_vals);
3435 }
3436}
3437
3438static void action_data_destroy(struct action_data *data)
3439{
3440 unsigned int i;
3441
3442 lockdep_assert_held(&event_mutex);
3443
3444 kfree(data->action_name);
3445
3446 for (i = 0; i < data->n_params; i++)
3447 kfree(data->params[i]);
3448
3449 if (data->synth_event)
3450 data->synth_event->ref--;
3451
3452 kfree(data->synth_event_name);
3453
3454 kfree(data);
3455}
3456
3457static void track_data_destroy(struct hist_trigger_data *hist_data,
3458 struct action_data *data)
3459{
3460 struct trace_event_file *file = hist_data->event_file;
3461
3462 destroy_hist_field(data->track_data.track_var, 0);
3463
3464 if (data->action == ACTION_SNAPSHOT) {
3465 struct track_data *track_data;
3466
3467 track_data = tracing_cond_snapshot_data(file->tr);
3468 if (track_data && track_data->hist_data == hist_data) {
3469 tracing_snapshot_cond_disable(file->tr);
3470 track_data_free(track_data);
3471 }
3472 }
3473
3474 kfree(data->track_data.var_str);
3475
3476 action_data_destroy(data);
3477}
3478
3479static int action_create(struct hist_trigger_data *hist_data,
3480 struct action_data *data);
3481
3482static int track_data_create(struct hist_trigger_data *hist_data,
3483 struct action_data *data)
3484{
3485 struct hist_field *var_field, *ref_field, *track_var = NULL;
3486 struct trace_event_file *file = hist_data->event_file;
3487 struct trace_array *tr = file->tr;
3488 char *track_data_var_str;
3489 int ret = 0;
3490
3491 track_data_var_str = data->track_data.var_str;
3492 if (track_data_var_str[0] != '$') {
3493 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3494 return -EINVAL;
3495 }
3496 track_data_var_str++;
3497
3498 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3499 if (!var_field) {
3500 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3501 return -EINVAL;
3502 }
3503
3504 ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3505 if (!ref_field)
3506 return -ENOMEM;
3507
3508 data->track_data.var_ref = ref_field;
3509
3510 if (data->handler == HANDLER_ONMAX)
3511 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3512 if (IS_ERR(track_var)) {
3513 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3514 ret = PTR_ERR(track_var);
3515 goto out;
3516 }
3517
3518 if (data->handler == HANDLER_ONCHANGE)
3519 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3520 if (IS_ERR(track_var)) {
3521 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3522 ret = PTR_ERR(track_var);
3523 goto out;
3524 }
3525 data->track_data.track_var = track_var;
3526
3527 ret = action_create(hist_data, data);
3528 out:
3529 return ret;
3530}
3531
3532static int parse_action_params(struct trace_array *tr, char *params,
3533 struct action_data *data)
3534{
3535 char *param, *saved_param;
3536 bool first_param = true;
3537 int ret = 0;
3538
3539 while (params) {
3540 if (data->n_params >= SYNTH_FIELDS_MAX) {
3541 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3542 goto out;
3543 }
3544
3545 param = strsep(¶ms, ",");
3546 if (!param) {
3547 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3548 ret = -EINVAL;
3549 goto out;
3550 }
3551
3552 param = strstrip(param);
3553 if (strlen(param) < 2) {
3554 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3555 ret = -EINVAL;
3556 goto out;
3557 }
3558
3559 saved_param = kstrdup(param, GFP_KERNEL);
3560 if (!saved_param) {
3561 ret = -ENOMEM;
3562 goto out;
3563 }
3564
3565 if (first_param && data->use_trace_keyword) {
3566 data->synth_event_name = saved_param;
3567 first_param = false;
3568 continue;
3569 }
3570 first_param = false;
3571
3572 data->params[data->n_params++] = saved_param;
3573 }
3574 out:
3575 return ret;
3576}
3577
3578static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3579 enum handler_id handler)
3580{
3581 char *action_name;
3582 int ret = 0;
3583
3584 strsep(&str, ".");
3585 if (!str) {
3586 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3587 ret = -EINVAL;
3588 goto out;
3589 }
3590
3591 action_name = strsep(&str, "(");
3592 if (!action_name || !str) {
3593 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3594 ret = -EINVAL;
3595 goto out;
3596 }
3597
3598 if (str_has_prefix(action_name, "save")) {
3599 char *params = strsep(&str, ")");
3600
3601 if (!params) {
3602 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3603 ret = -EINVAL;
3604 goto out;
3605 }
3606
3607 ret = parse_action_params(tr, params, data);
3608 if (ret)
3609 goto out;
3610
3611 if (handler == HANDLER_ONMAX)
3612 data->track_data.check_val = check_track_val_max;
3613 else if (handler == HANDLER_ONCHANGE)
3614 data->track_data.check_val = check_track_val_changed;
3615 else {
3616 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3617 ret = -EINVAL;
3618 goto out;
3619 }
3620
3621 data->track_data.save_data = save_track_data_vars;
3622 data->fn = ontrack_action;
3623 data->action = ACTION_SAVE;
3624 } else if (str_has_prefix(action_name, "snapshot")) {
3625 char *params = strsep(&str, ")");
3626
3627 if (!str) {
3628 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3629 ret = -EINVAL;
3630 goto out;
3631 }
3632
3633 if (handler == HANDLER_ONMAX)
3634 data->track_data.check_val = check_track_val_max;
3635 else if (handler == HANDLER_ONCHANGE)
3636 data->track_data.check_val = check_track_val_changed;
3637 else {
3638 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3639 ret = -EINVAL;
3640 goto out;
3641 }
3642
3643 data->track_data.save_data = save_track_data_snapshot;
3644 data->fn = ontrack_action;
3645 data->action = ACTION_SNAPSHOT;
3646 } else {
3647 char *params = strsep(&str, ")");
3648
3649 if (str_has_prefix(action_name, "trace"))
3650 data->use_trace_keyword = true;
3651
3652 if (params) {
3653 ret = parse_action_params(tr, params, data);
3654 if (ret)
3655 goto out;
3656 }
3657
3658 if (handler == HANDLER_ONMAX)
3659 data->track_data.check_val = check_track_val_max;
3660 else if (handler == HANDLER_ONCHANGE)
3661 data->track_data.check_val = check_track_val_changed;
3662
3663 if (handler != HANDLER_ONMATCH) {
3664 data->track_data.save_data = action_trace;
3665 data->fn = ontrack_action;
3666 } else
3667 data->fn = action_trace;
3668
3669 data->action = ACTION_TRACE;
3670 }
3671
3672 data->action_name = kstrdup(action_name, GFP_KERNEL);
3673 if (!data->action_name) {
3674 ret = -ENOMEM;
3675 goto out;
3676 }
3677
3678 data->handler = handler;
3679 out:
3680 return ret;
3681}
3682
3683static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3684 char *str, enum handler_id handler)
3685{
3686 struct action_data *data;
3687 int ret = -EINVAL;
3688 char *var_str;
3689
3690 data = kzalloc(sizeof(*data), GFP_KERNEL);
3691 if (!data)
3692 return ERR_PTR(-ENOMEM);
3693
3694 var_str = strsep(&str, ")");
3695 if (!var_str || !str) {
3696 ret = -EINVAL;
3697 goto free;
3698 }
3699
3700 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3701 if (!data->track_data.var_str) {
3702 ret = -ENOMEM;
3703 goto free;
3704 }
3705
3706 ret = action_parse(hist_data->event_file->tr, str, data, handler);
3707 if (ret)
3708 goto free;
3709 out:
3710 return data;
3711 free:
3712 track_data_destroy(hist_data, data);
3713 data = ERR_PTR(ret);
3714 goto out;
3715}
3716
3717static void onmatch_destroy(struct action_data *data)
3718{
3719 kfree(data->match_data.event);
3720 kfree(data->match_data.event_system);
3721
3722 action_data_destroy(data);
3723}
3724
3725static void destroy_field_var(struct field_var *field_var)
3726{
3727 if (!field_var)
3728 return;
3729
3730 destroy_hist_field(field_var->var, 0);
3731 destroy_hist_field(field_var->val, 0);
3732
3733 kfree(field_var);
3734}
3735
3736static void destroy_field_vars(struct hist_trigger_data *hist_data)
3737{
3738 unsigned int i;
3739
3740 for (i = 0; i < hist_data->n_field_vars; i++)
3741 destroy_field_var(hist_data->field_vars[i]);
3742
3743 for (i = 0; i < hist_data->n_save_vars; i++)
3744 destroy_field_var(hist_data->save_vars[i]);
3745}
3746
3747static void save_field_var(struct hist_trigger_data *hist_data,
3748 struct field_var *field_var)
3749{
3750 hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3751
3752 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3753 hist_data->n_field_var_str++;
3754}
3755
3756
3757static int check_synth_field(struct synth_event *event,
3758 struct hist_field *hist_field,
3759 unsigned int field_pos)
3760{
3761 struct synth_field *field;
3762
3763 if (field_pos >= event->n_fields)
3764 return -EINVAL;
3765
3766 field = event->fields[field_pos];
3767
3768
3769
3770
3771
3772
3773 if (strstr(hist_field->type, "char[") && field->is_string
3774 && field->is_dynamic)
3775 return 0;
3776
3777 if (strcmp(field->type, hist_field->type) != 0) {
3778 if (field->size != hist_field->size ||
3779 (!field->is_string && field->is_signed != hist_field->is_signed))
3780 return -EINVAL;
3781 }
3782
3783 return 0;
3784}
3785
3786static struct hist_field *
3787trace_action_find_var(struct hist_trigger_data *hist_data,
3788 struct action_data *data,
3789 char *system, char *event, char *var)
3790{
3791 struct trace_array *tr = hist_data->event_file->tr;
3792 struct hist_field *hist_field;
3793
3794 var++;
3795
3796 hist_field = find_target_event_var(hist_data, system, event, var);
3797 if (!hist_field) {
3798 if (!system && data->handler == HANDLER_ONMATCH) {
3799 system = data->match_data.event_system;
3800 event = data->match_data.event;
3801 }
3802
3803 hist_field = find_event_var(hist_data, system, event, var);
3804 }
3805
3806 if (!hist_field)
3807 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3808
3809 return hist_field;
3810}
3811
3812static struct hist_field *
3813trace_action_create_field_var(struct hist_trigger_data *hist_data,
3814 struct action_data *data, char *system,
3815 char *event, char *var)
3816{
3817 struct hist_field *hist_field = NULL;
3818 struct field_var *field_var;
3819
3820
3821
3822
3823
3824
3825
3826 field_var = create_target_field_var(hist_data, system, event, var);
3827
3828 if (field_var && !IS_ERR(field_var)) {
3829 save_field_var(hist_data, field_var);
3830 hist_field = field_var->var;
3831 } else {
3832 field_var = NULL;
3833
3834
3835
3836
3837
3838 if (!system && data->handler == HANDLER_ONMATCH) {
3839 system = data->match_data.event_system;
3840 event = data->match_data.event;
3841 }
3842
3843 if (!event)
3844 goto free;
3845
3846
3847
3848
3849
3850
3851
3852 hist_field = create_field_var_hist(hist_data, system, event, var);
3853 if (IS_ERR(hist_field))
3854 goto free;
3855 }
3856 out:
3857 return hist_field;
3858 free:
3859 destroy_field_var(field_var);
3860 hist_field = NULL;
3861 goto out;
3862}
3863
3864static int trace_action_create(struct hist_trigger_data *hist_data,
3865 struct action_data *data)
3866{
3867 struct trace_array *tr = hist_data->event_file->tr;
3868 char *event_name, *param, *system = NULL;
3869 struct hist_field *hist_field, *var_ref;
3870 unsigned int i;
3871 unsigned int field_pos = 0;
3872 struct synth_event *event;
3873 char *synth_event_name;
3874 int var_ref_idx, ret = 0;
3875
3876 lockdep_assert_held(&event_mutex);
3877
3878 if (data->use_trace_keyword)
3879 synth_event_name = data->synth_event_name;
3880 else
3881 synth_event_name = data->action_name;
3882
3883 event = find_synth_event(synth_event_name);
3884 if (!event) {
3885 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3886 return -EINVAL;
3887 }
3888
3889 event->ref++;
3890
3891 for (i = 0; i < data->n_params; i++) {
3892 char *p;
3893
3894 p = param = kstrdup(data->params[i], GFP_KERNEL);
3895 if (!param) {
3896 ret = -ENOMEM;
3897 goto err;
3898 }
3899
3900 system = strsep(¶m, ".");
3901 if (!param) {
3902 param = (char *)system;
3903 system = event_name = NULL;
3904 } else {
3905 event_name = strsep(¶m, ".");
3906 if (!param) {
3907 kfree(p);
3908 ret = -EINVAL;
3909 goto err;
3910 }
3911 }
3912
3913 if (param[0] == '$')
3914 hist_field = trace_action_find_var(hist_data, data,
3915 system, event_name,
3916 param);
3917 else
3918 hist_field = trace_action_create_field_var(hist_data,
3919 data,
3920 system,
3921 event_name,
3922 param);
3923
3924 if (!hist_field) {
3925 kfree(p);
3926 ret = -EINVAL;
3927 goto err;
3928 }
3929
3930 if (check_synth_field(event, hist_field, field_pos) == 0) {
3931 var_ref = create_var_ref(hist_data, hist_field,
3932 system, event_name);
3933 if (!var_ref) {
3934 kfree(p);
3935 ret = -ENOMEM;
3936 goto err;
3937 }
3938
3939 var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3940 if (WARN_ON(var_ref_idx < 0)) {
3941 kfree(p);
3942 ret = var_ref_idx;
3943 goto err;
3944 }
3945
3946 data->var_ref_idx[i] = var_ref_idx;
3947
3948 field_pos++;
3949 kfree(p);
3950 continue;
3951 }
3952
3953 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3954 kfree(p);
3955 ret = -EINVAL;
3956 goto err;
3957 }
3958
3959 if (field_pos != event->n_fields) {
3960 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3961 ret = -EINVAL;
3962 goto err;
3963 }
3964
3965 data->synth_event = event;
3966 out:
3967 return ret;
3968 err:
3969 event->ref--;
3970
3971 goto out;
3972}
3973
3974static int action_create(struct hist_trigger_data *hist_data,
3975 struct action_data *data)
3976{
3977 struct trace_event_file *file = hist_data->event_file;
3978 struct trace_array *tr = file->tr;
3979 struct track_data *track_data;
3980 struct field_var *field_var;
3981 unsigned int i;
3982 char *param;
3983 int ret = 0;
3984
3985 if (data->action == ACTION_TRACE)
3986 return trace_action_create(hist_data, data);
3987
3988 if (data->action == ACTION_SNAPSHOT) {
3989 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3990 if (IS_ERR(track_data)) {
3991 ret = PTR_ERR(track_data);
3992 goto out;
3993 }
3994
3995 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3996 cond_snapshot_update);
3997 if (ret)
3998 track_data_free(track_data);
3999
4000 goto out;
4001 }
4002
4003 if (data->action == ACTION_SAVE) {
4004 if (hist_data->n_save_vars) {
4005 ret = -EEXIST;
4006 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4007 goto out;
4008 }
4009
4010 for (i = 0; i < data->n_params; i++) {
4011 param = kstrdup(data->params[i], GFP_KERNEL);
4012 if (!param) {
4013 ret = -ENOMEM;
4014 goto out;
4015 }
4016
4017 field_var = create_target_field_var(hist_data, NULL, NULL, param);
4018 if (IS_ERR(field_var)) {
4019 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4020 errpos(param));
4021 ret = PTR_ERR(field_var);
4022 kfree(param);
4023 goto out;
4024 }
4025
4026 hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4027 if (field_var->val->flags & HIST_FIELD_FL_STRING)
4028 hist_data->n_save_var_str++;
4029 kfree(param);
4030 }
4031 }
4032 out:
4033 return ret;
4034}
4035
4036static int onmatch_create(struct hist_trigger_data *hist_data,
4037 struct action_data *data)
4038{
4039 return action_create(hist_data, data);
4040}
4041
4042static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4043{
4044 char *match_event, *match_event_system;
4045 struct action_data *data;
4046 int ret = -EINVAL;
4047
4048 data = kzalloc(sizeof(*data), GFP_KERNEL);
4049 if (!data)
4050 return ERR_PTR(-ENOMEM);
4051
4052 match_event = strsep(&str, ")");
4053 if (!match_event || !str) {
4054 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4055 goto free;
4056 }
4057
4058 match_event_system = strsep(&match_event, ".");
4059 if (!match_event) {
4060 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4061 goto free;
4062 }
4063
4064 if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4065 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4066 goto free;
4067 }
4068
4069 data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4070 if (!data->match_data.event) {
4071 ret = -ENOMEM;
4072 goto free;
4073 }
4074
4075 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4076 if (!data->match_data.event_system) {
4077 ret = -ENOMEM;
4078 goto free;
4079 }
4080
4081 ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4082 if (ret)
4083 goto free;
4084 out:
4085 return data;
4086 free:
4087 onmatch_destroy(data);
4088 data = ERR_PTR(ret);
4089 goto out;
4090}
4091
4092static int create_hitcount_val(struct hist_trigger_data *hist_data)
4093{
4094 hist_data->fields[HITCOUNT_IDX] =
4095 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4096 if (!hist_data->fields[HITCOUNT_IDX])
4097 return -ENOMEM;
4098
4099 hist_data->n_vals++;
4100 hist_data->n_fields++;
4101
4102 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4103 return -EINVAL;
4104
4105 return 0;
4106}
4107
4108static int __create_val_field(struct hist_trigger_data *hist_data,
4109 unsigned int val_idx,
4110 struct trace_event_file *file,
4111 char *var_name, char *field_str,
4112 unsigned long flags)
4113{
4114 struct hist_field *hist_field;
4115 int ret = 0, n_subexprs = 0;
4116
4117 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4118 if (IS_ERR(hist_field)) {
4119 ret = PTR_ERR(hist_field);
4120 goto out;
4121 }
4122
4123 hist_data->fields[val_idx] = hist_field;
4124
4125 ++hist_data->n_vals;
4126 ++hist_data->n_fields;
4127
4128 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4129 ret = -EINVAL;
4130 out:
4131 return ret;
4132}
4133
4134static int create_val_field(struct hist_trigger_data *hist_data,
4135 unsigned int val_idx,
4136 struct trace_event_file *file,
4137 char *field_str)
4138{
4139 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4140 return -EINVAL;
4141
4142 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4143}
4144
4145static const char *no_comm = "(no comm)";
4146
4147static u64 hist_field_execname(struct hist_field *hist_field,
4148 struct tracing_map_elt *elt,
4149 struct trace_buffer *buffer,
4150 struct ring_buffer_event *rbe,
4151 void *event)
4152{
4153 struct hist_elt_data *elt_data;
4154
4155 if (WARN_ON_ONCE(!elt))
4156 return (u64)(unsigned long)no_comm;
4157
4158 elt_data = elt->private_data;
4159
4160 if (WARN_ON_ONCE(!elt_data->comm))
4161 return (u64)(unsigned long)no_comm;
4162
4163 return (u64)(unsigned long)(elt_data->comm);
4164}
4165
4166
4167static void update_var_execname(struct hist_field *hist_field)
4168{
4169 hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4170 HIST_FIELD_FL_EXECNAME;
4171 hist_field->size = MAX_FILTER_STR_VAL;
4172 hist_field->is_signed = 0;
4173
4174 kfree_const(hist_field->type);
4175 hist_field->type = "char[]";
4176
4177 hist_field->fn = hist_field_execname;
4178}
4179
4180static int create_var_field(struct hist_trigger_data *hist_data,
4181 unsigned int val_idx,
4182 struct trace_event_file *file,
4183 char *var_name, char *expr_str)
4184{
4185 struct trace_array *tr = hist_data->event_file->tr;
4186 unsigned long flags = 0;
4187 int ret;
4188
4189 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4190 return -EINVAL;
4191
4192 if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4193 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4194 return -EINVAL;
4195 }
4196
4197 flags |= HIST_FIELD_FL_VAR;
4198 hist_data->n_vars++;
4199 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4200 return -EINVAL;
4201
4202 ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4203
4204 if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4205 update_var_execname(hist_data->fields[val_idx]);
4206
4207 if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
4208 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4209
4210 return ret;
4211}
4212
4213static int create_val_fields(struct hist_trigger_data *hist_data,
4214 struct trace_event_file *file)
4215{
4216 char *fields_str, *field_str;
4217 unsigned int i, j = 1;
4218 int ret;
4219
4220 ret = create_hitcount_val(hist_data);
4221 if (ret)
4222 goto out;
4223
4224 fields_str = hist_data->attrs->vals_str;
4225 if (!fields_str)
4226 goto out;
4227
4228 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4229 j < TRACING_MAP_VALS_MAX; i++) {
4230 field_str = strsep(&fields_str, ",");
4231 if (!field_str)
4232 break;
4233
4234 if (strcmp(field_str, "hitcount") == 0)
4235 continue;
4236
4237 ret = create_val_field(hist_data, j++, file, field_str);
4238 if (ret)
4239 goto out;
4240 }
4241
4242 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4243 ret = -EINVAL;
4244 out:
4245 return ret;
4246}
4247
4248static int create_key_field(struct hist_trigger_data *hist_data,
4249 unsigned int key_idx,
4250 unsigned int key_offset,
4251 struct trace_event_file *file,
4252 char *field_str)
4253{
4254 struct trace_array *tr = hist_data->event_file->tr;
4255 struct hist_field *hist_field = NULL;
4256 unsigned long flags = 0;
4257 unsigned int key_size;
4258 int ret = 0, n_subexprs = 0;
4259
4260 if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4261 return -EINVAL;
4262
4263 flags |= HIST_FIELD_FL_KEY;
4264
4265 if (strcmp(field_str, "stacktrace") == 0) {
4266 flags |= HIST_FIELD_FL_STACKTRACE;
4267 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4268 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4269 } else {
4270 hist_field = parse_expr(hist_data, file, field_str, flags,
4271 NULL, &n_subexprs);
4272 if (IS_ERR(hist_field)) {
4273 ret = PTR_ERR(hist_field);
4274 goto out;
4275 }
4276
4277 if (field_has_hist_vars(hist_field, 0)) {
4278 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4279 destroy_hist_field(hist_field, 0);
4280 ret = -EINVAL;
4281 goto out;
4282 }
4283
4284 key_size = hist_field->size;
4285 }
4286
4287 hist_data->fields[key_idx] = hist_field;
4288
4289 key_size = ALIGN(key_size, sizeof(u64));
4290 hist_data->fields[key_idx]->size = key_size;
4291 hist_data->fields[key_idx]->offset = key_offset;
4292
4293 hist_data->key_size += key_size;
4294
4295 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4296 ret = -EINVAL;
4297 goto out;
4298 }
4299
4300 hist_data->n_keys++;
4301 hist_data->n_fields++;
4302
4303 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4304 return -EINVAL;
4305
4306 ret = key_size;
4307 out:
4308 return ret;
4309}
4310
4311static int create_key_fields(struct hist_trigger_data *hist_data,
4312 struct trace_event_file *file)
4313{
4314 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4315 char *fields_str, *field_str;
4316 int ret = -EINVAL;
4317
4318 fields_str = hist_data->attrs->keys_str;
4319 if (!fields_str)
4320 goto out;
4321
4322 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4323 field_str = strsep(&fields_str, ",");
4324 if (!field_str)
4325 break;
4326 ret = create_key_field(hist_data, i, key_offset,
4327 file, field_str);
4328 if (ret < 0)
4329 goto out;
4330 key_offset += ret;
4331 }
4332 if (fields_str) {
4333 ret = -EINVAL;
4334 goto out;
4335 }
4336 ret = 0;
4337 out:
4338 return ret;
4339}
4340
4341static int create_var_fields(struct hist_trigger_data *hist_data,
4342 struct trace_event_file *file)
4343{
4344 unsigned int i, j = hist_data->n_vals;
4345 int ret = 0;
4346
4347 unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4348
4349 for (i = 0; i < n_vars; i++) {
4350 char *var_name = hist_data->attrs->var_defs.name[i];
4351 char *expr = hist_data->attrs->var_defs.expr[i];
4352
4353 ret = create_var_field(hist_data, j++, file, var_name, expr);
4354 if (ret)
4355 goto out;
4356 }
4357 out:
4358 return ret;
4359}
4360
4361static void free_var_defs(struct hist_trigger_data *hist_data)
4362{
4363 unsigned int i;
4364
4365 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4366 kfree(hist_data->attrs->var_defs.name[i]);
4367 kfree(hist_data->attrs->var_defs.expr[i]);
4368 }
4369
4370 hist_data->attrs->var_defs.n_vars = 0;
4371}
4372
4373static int parse_var_defs(struct hist_trigger_data *hist_data)
4374{
4375 struct trace_array *tr = hist_data->event_file->tr;
4376 char *s, *str, *var_name, *field_str;
4377 unsigned int i, j, n_vars = 0;
4378 int ret = 0;
4379
4380 for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4381 str = hist_data->attrs->assignment_str[i];
4382 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4383 field_str = strsep(&str, ",");
4384 if (!field_str)
4385 break;
4386
4387 var_name = strsep(&field_str, "=");
4388 if (!var_name || !field_str) {
4389 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4390 errpos(var_name));
4391 ret = -EINVAL;
4392 goto free;
4393 }
4394
4395 if (n_vars == TRACING_MAP_VARS_MAX) {
4396 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4397 ret = -EINVAL;
4398 goto free;
4399 }
4400
4401 s = kstrdup(var_name, GFP_KERNEL);
4402 if (!s) {
4403 ret = -ENOMEM;
4404 goto free;
4405 }
4406 hist_data->attrs->var_defs.name[n_vars] = s;
4407
4408 s = kstrdup(field_str, GFP_KERNEL);
4409 if (!s) {
4410 ret = -ENOMEM;
4411 goto free;
4412 }
4413 hist_data->attrs->var_defs.expr[n_vars++] = s;
4414
4415 hist_data->attrs->var_defs.n_vars = n_vars;
4416 }
4417 }
4418
4419 return ret;
4420 free:
4421 free_var_defs(hist_data);
4422
4423 return ret;
4424}
4425
4426static int create_hist_fields(struct hist_trigger_data *hist_data,
4427 struct trace_event_file *file)
4428{
4429 int ret;
4430
4431 ret = parse_var_defs(hist_data);
4432 if (ret)
4433 goto out;
4434
4435 ret = create_val_fields(hist_data, file);
4436 if (ret)
4437 goto out;
4438
4439 ret = create_var_fields(hist_data, file);
4440 if (ret)
4441 goto out;
4442
4443 ret = create_key_fields(hist_data, file);
4444 if (ret)
4445 goto out;
4446 out:
4447 free_var_defs(hist_data);
4448
4449 return ret;
4450}
4451
4452static int is_descending(struct trace_array *tr, const char *str)
4453{
4454 if (!str)
4455 return 0;
4456
4457 if (strcmp(str, "descending") == 0)
4458 return 1;
4459
4460 if (strcmp(str, "ascending") == 0)
4461 return 0;
4462
4463 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4464
4465 return -EINVAL;
4466}
4467
4468static int create_sort_keys(struct hist_trigger_data *hist_data)
4469{
4470 struct trace_array *tr = hist_data->event_file->tr;
4471 char *fields_str = hist_data->attrs->sort_key_str;
4472 struct tracing_map_sort_key *sort_key;
4473 int descending, ret = 0;
4474 unsigned int i, j, k;
4475
4476 hist_data->n_sort_keys = 1;
4477
4478 if (!fields_str)
4479 goto out;
4480
4481 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4482 struct hist_field *hist_field;
4483 char *field_str, *field_name;
4484 const char *test_name;
4485
4486 sort_key = &hist_data->sort_keys[i];
4487
4488 field_str = strsep(&fields_str, ",");
4489 if (!field_str)
4490 break;
4491
4492 if (!*field_str) {
4493 ret = -EINVAL;
4494 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4495 break;
4496 }
4497
4498 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4499 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4500 ret = -EINVAL;
4501 break;
4502 }
4503
4504 field_name = strsep(&field_str, ".");
4505 if (!field_name || !*field_name) {
4506 ret = -EINVAL;
4507 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4508 break;
4509 }
4510
4511 if (strcmp(field_name, "hitcount") == 0) {
4512 descending = is_descending(tr, field_str);
4513 if (descending < 0) {
4514 ret = descending;
4515 break;
4516 }
4517 sort_key->descending = descending;
4518 continue;
4519 }
4520
4521 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4522 unsigned int idx;
4523
4524 hist_field = hist_data->fields[j];
4525 if (hist_field->flags & HIST_FIELD_FL_VAR)
4526 continue;
4527
4528 idx = k++;
4529
4530 test_name = hist_field_name(hist_field, 0);
4531
4532 if (strcmp(field_name, test_name) == 0) {
4533 sort_key->field_idx = idx;
4534 descending = is_descending(tr, field_str);
4535 if (descending < 0) {
4536 ret = descending;
4537 goto out;
4538 }
4539 sort_key->descending = descending;
4540 break;
4541 }
4542 }
4543 if (j == hist_data->n_fields) {
4544 ret = -EINVAL;
4545 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4546 break;
4547 }
4548 }
4549
4550 hist_data->n_sort_keys = i;
4551 out:
4552 return ret;
4553}
4554
4555static void destroy_actions(struct hist_trigger_data *hist_data)
4556{
4557 unsigned int i;
4558
4559 for (i = 0; i < hist_data->n_actions; i++) {
4560 struct action_data *data = hist_data->actions[i];
4561
4562 if (data->handler == HANDLER_ONMATCH)
4563 onmatch_destroy(data);
4564 else if (data->handler == HANDLER_ONMAX ||
4565 data->handler == HANDLER_ONCHANGE)
4566 track_data_destroy(hist_data, data);
4567 else
4568 kfree(data);
4569 }
4570}
4571
4572static int parse_actions(struct hist_trigger_data *hist_data)
4573{
4574 struct trace_array *tr = hist_data->event_file->tr;
4575 struct action_data *data;
4576 unsigned int i;
4577 int ret = 0;
4578 char *str;
4579 int len;
4580
4581 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4582 str = hist_data->attrs->action_str[i];
4583
4584 if ((len = str_has_prefix(str, "onmatch("))) {
4585 char *action_str = str + len;
4586
4587 data = onmatch_parse(tr, action_str);
4588 if (IS_ERR(data)) {
4589 ret = PTR_ERR(data);
4590 break;
4591 }
4592 } else if ((len = str_has_prefix(str, "onmax("))) {
4593 char *action_str = str + len;
4594
4595 data = track_data_parse(hist_data, action_str,
4596 HANDLER_ONMAX);
4597 if (IS_ERR(data)) {
4598 ret = PTR_ERR(data);
4599 break;
4600 }
4601 } else if ((len = str_has_prefix(str, "onchange("))) {
4602 char *action_str = str + len;
4603
4604 data = track_data_parse(hist_data, action_str,
4605 HANDLER_ONCHANGE);
4606 if (IS_ERR(data)) {
4607 ret = PTR_ERR(data);
4608 break;
4609 }
4610 } else {
4611 ret = -EINVAL;
4612 break;
4613 }
4614
4615 hist_data->actions[hist_data->n_actions++] = data;
4616 }
4617
4618 return ret;
4619}
4620
4621static int create_actions(struct hist_trigger_data *hist_data)
4622{
4623 struct action_data *data;
4624 unsigned int i;
4625 int ret = 0;
4626
4627 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4628 data = hist_data->actions[i];
4629
4630 if (data->handler == HANDLER_ONMATCH) {
4631 ret = onmatch_create(hist_data, data);
4632 if (ret)
4633 break;
4634 } else if (data->handler == HANDLER_ONMAX ||
4635 data->handler == HANDLER_ONCHANGE) {
4636 ret = track_data_create(hist_data, data);
4637 if (ret)
4638 break;
4639 } else {
4640 ret = -EINVAL;
4641 break;
4642 }
4643 }
4644
4645 return ret;
4646}
4647
4648static void print_actions(struct seq_file *m,
4649 struct hist_trigger_data *hist_data,
4650 struct tracing_map_elt *elt)
4651{
4652 unsigned int i;
4653
4654 for (i = 0; i < hist_data->n_actions; i++) {
4655 struct action_data *data = hist_data->actions[i];
4656
4657 if (data->action == ACTION_SNAPSHOT)
4658 continue;
4659
4660 if (data->handler == HANDLER_ONMAX ||
4661 data->handler == HANDLER_ONCHANGE)
4662 track_data_print(m, hist_data, elt, data);
4663 }
4664}
4665
4666static void print_action_spec(struct seq_file *m,
4667 struct hist_trigger_data *hist_data,
4668 struct action_data *data)
4669{
4670 unsigned int i;
4671
4672 if (data->action == ACTION_SAVE) {
4673 for (i = 0; i < hist_data->n_save_vars; i++) {
4674 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4675 if (i < hist_data->n_save_vars - 1)
4676 seq_puts(m, ",");
4677 }
4678 } else if (data->action == ACTION_TRACE) {
4679 if (data->use_trace_keyword)
4680 seq_printf(m, "%s", data->synth_event_name);
4681 for (i = 0; i < data->n_params; i++) {
4682 if (i || data->use_trace_keyword)
4683 seq_puts(m, ",");
4684 seq_printf(m, "%s", data->params[i]);
4685 }
4686 }
4687}
4688
4689static void print_track_data_spec(struct seq_file *m,
4690 struct hist_trigger_data *hist_data,
4691 struct action_data *data)
4692{
4693 if (data->handler == HANDLER_ONMAX)
4694 seq_puts(m, ":onmax(");
4695 else if (data->handler == HANDLER_ONCHANGE)
4696 seq_puts(m, ":onchange(");
4697 seq_printf(m, "%s", data->track_data.var_str);
4698 seq_printf(m, ").%s(", data->action_name);
4699
4700 print_action_spec(m, hist_data, data);
4701
4702 seq_puts(m, ")");
4703}
4704
4705static void print_onmatch_spec(struct seq_file *m,
4706 struct hist_trigger_data *hist_data,
4707 struct action_data *data)
4708{
4709 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4710 data->match_data.event);
4711
4712 seq_printf(m, "%s(", data->action_name);
4713
4714 print_action_spec(m, hist_data, data);
4715
4716 seq_puts(m, ")");
4717}
4718
4719static bool actions_match(struct hist_trigger_data *hist_data,
4720 struct hist_trigger_data *hist_data_test)
4721{
4722 unsigned int i, j;
4723
4724 if (hist_data->n_actions != hist_data_test->n_actions)
4725 return false;
4726
4727 for (i = 0; i < hist_data->n_actions; i++) {
4728 struct action_data *data = hist_data->actions[i];
4729 struct action_data *data_test = hist_data_test->actions[i];
4730 char *action_name, *action_name_test;
4731
4732 if (data->handler != data_test->handler)
4733 return false;
4734 if (data->action != data_test->action)
4735 return false;
4736
4737 if (data->n_params != data_test->n_params)
4738 return false;
4739
4740 for (j = 0; j < data->n_params; j++) {
4741 if (strcmp(data->params[j], data_test->params[j]) != 0)
4742 return false;
4743 }
4744
4745 if (data->use_trace_keyword)
4746 action_name = data->synth_event_name;
4747 else
4748 action_name = data->action_name;
4749
4750 if (data_test->use_trace_keyword)
4751 action_name_test = data_test->synth_event_name;
4752 else
4753 action_name_test = data_test->action_name;
4754
4755 if (strcmp(action_name, action_name_test) != 0)
4756 return false;
4757
4758 if (data->handler == HANDLER_ONMATCH) {
4759 if (strcmp(data->match_data.event_system,
4760 data_test->match_data.event_system) != 0)
4761 return false;
4762 if (strcmp(data->match_data.event,
4763 data_test->match_data.event) != 0)
4764 return false;
4765 } else if (data->handler == HANDLER_ONMAX ||
4766 data->handler == HANDLER_ONCHANGE) {
4767 if (strcmp(data->track_data.var_str,
4768 data_test->track_data.var_str) != 0)
4769 return false;
4770 }
4771 }
4772
4773 return true;
4774}
4775
4776
4777static void print_actions_spec(struct seq_file *m,
4778 struct hist_trigger_data *hist_data)
4779{
4780 unsigned int i;
4781
4782 for (i = 0; i < hist_data->n_actions; i++) {
4783 struct action_data *data = hist_data->actions[i];
4784
4785 if (data->handler == HANDLER_ONMATCH)
4786 print_onmatch_spec(m, hist_data, data);
4787 else if (data->handler == HANDLER_ONMAX ||
4788 data->handler == HANDLER_ONCHANGE)
4789 print_track_data_spec(m, hist_data, data);
4790 }
4791}
4792
4793static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4794{
4795 unsigned int i;
4796
4797 for (i = 0; i < hist_data->n_field_var_hists; i++) {
4798 kfree(hist_data->field_var_hists[i]->cmd);
4799 kfree(hist_data->field_var_hists[i]);
4800 }
4801}
4802
4803static void destroy_hist_data(struct hist_trigger_data *hist_data)
4804{
4805 if (!hist_data)
4806 return;
4807
4808 destroy_hist_trigger_attrs(hist_data->attrs);
4809 destroy_hist_fields(hist_data);
4810 tracing_map_destroy(hist_data->map);
4811
4812 destroy_actions(hist_data);
4813 destroy_field_vars(hist_data);
4814 destroy_field_var_hists(hist_data);
4815
4816 kfree(hist_data);
4817}
4818
4819static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4820{
4821 struct tracing_map *map = hist_data->map;
4822 struct ftrace_event_field *field;
4823 struct hist_field *hist_field;
4824 int i, idx = 0;
4825
4826 for_each_hist_field(i, hist_data) {
4827 hist_field = hist_data->fields[i];
4828 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4829 tracing_map_cmp_fn_t cmp_fn;
4830
4831 field = hist_field->field;
4832
4833 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4834 cmp_fn = tracing_map_cmp_none;
4835 else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
4836 cmp_fn = tracing_map_cmp_num(hist_field->size,
4837 hist_field->is_signed);
4838 else if (is_string_field(field))
4839 cmp_fn = tracing_map_cmp_string;
4840 else
4841 cmp_fn = tracing_map_cmp_num(field->size,
4842 field->is_signed);
4843 idx = tracing_map_add_key_field(map,
4844 hist_field->offset,
4845 cmp_fn);
4846 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4847 idx = tracing_map_add_sum_field(map);
4848
4849 if (idx < 0)
4850 return idx;
4851
4852 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4853 idx = tracing_map_add_var(map);
4854 if (idx < 0)
4855 return idx;
4856 hist_field->var.idx = idx;
4857 hist_field->var.hist_data = hist_data;
4858 }
4859 }
4860
4861 return 0;
4862}
4863
4864static struct hist_trigger_data *
4865create_hist_data(unsigned int map_bits,
4866 struct hist_trigger_attrs *attrs,
4867 struct trace_event_file *file,
4868 bool remove)
4869{
4870 const struct tracing_map_ops *map_ops = NULL;
4871 struct hist_trigger_data *hist_data;
4872 int ret = 0;
4873
4874 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4875 if (!hist_data)
4876 return ERR_PTR(-ENOMEM);
4877
4878 hist_data->attrs = attrs;
4879 hist_data->remove = remove;
4880 hist_data->event_file = file;
4881
4882 ret = parse_actions(hist_data);
4883 if (ret)
4884 goto free;
4885
4886 ret = create_hist_fields(hist_data, file);
4887 if (ret)
4888 goto free;
4889
4890 ret = create_sort_keys(hist_data);
4891 if (ret)
4892 goto free;
4893
4894 map_ops = &hist_trigger_elt_data_ops;
4895
4896 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4897 map_ops, hist_data);
4898 if (IS_ERR(hist_data->map)) {
4899 ret = PTR_ERR(hist_data->map);
4900 hist_data->map = NULL;
4901 goto free;
4902 }
4903
4904 ret = create_tracing_map_fields(hist_data);
4905 if (ret)
4906 goto free;
4907 out:
4908 return hist_data;
4909 free:
4910 hist_data->attrs = NULL;
4911
4912 destroy_hist_data(hist_data);
4913
4914 hist_data = ERR_PTR(ret);
4915
4916 goto out;
4917}
4918
4919static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4920 struct tracing_map_elt *elt,
4921 struct trace_buffer *buffer, void *rec,
4922 struct ring_buffer_event *rbe,
4923 u64 *var_ref_vals)
4924{
4925 struct hist_elt_data *elt_data;
4926 struct hist_field *hist_field;
4927 unsigned int i, var_idx;
4928 u64 hist_val;
4929
4930 elt_data = elt->private_data;
4931 elt_data->var_ref_vals = var_ref_vals;
4932
4933 for_each_hist_val_field(i, hist_data) {
4934 hist_field = hist_data->fields[i];
4935 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4936 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4937 var_idx = hist_field->var.idx;
4938
4939 if (hist_field->flags & HIST_FIELD_FL_STRING) {
4940 unsigned int str_start, var_str_idx, idx;
4941 char *str, *val_str;
4942 unsigned int size;
4943
4944 str_start = hist_data->n_field_var_str +
4945 hist_data->n_save_var_str;
4946 var_str_idx = hist_field->var_str_idx;
4947 idx = str_start + var_str_idx;
4948
4949 str = elt_data->field_var_str[idx];
4950 val_str = (char *)(uintptr_t)hist_val;
4951
4952 size = min(hist_field->size, STR_VAR_LEN_MAX);
4953 strscpy(str, val_str, size);
4954
4955 hist_val = (u64)(uintptr_t)str;
4956 }
4957 tracing_map_set_var(elt, var_idx, hist_val);
4958 continue;
4959 }
4960 tracing_map_update_sum(elt, i, hist_val);
4961 }
4962
4963 for_each_hist_key_field(i, hist_data) {
4964 hist_field = hist_data->fields[i];
4965 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4966 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4967 var_idx = hist_field->var.idx;
4968 tracing_map_set_var(elt, var_idx, hist_val);
4969 }
4970 }
4971
4972 update_field_vars(hist_data, elt, buffer, rbe, rec);
4973}
4974
4975static inline void add_to_key(char *compound_key, void *key,
4976 struct hist_field *key_field, void *rec)
4977{
4978 size_t size = key_field->size;
4979
4980 if (key_field->flags & HIST_FIELD_FL_STRING) {
4981 struct ftrace_event_field *field;
4982
4983 field = key_field->field;
4984 if (field->filter_type == FILTER_DYN_STRING ||
4985 field->filter_type == FILTER_RDYN_STRING)
4986 size = *(u32 *)(rec + field->offset) >> 16;
4987 else if (field->filter_type == FILTER_STATIC_STRING)
4988 size = field->size;
4989
4990
4991 if (size > key_field->size - 1)
4992 size = key_field->size - 1;
4993
4994 strncpy(compound_key + key_field->offset, (char *)key, size);
4995 } else
4996 memcpy(compound_key + key_field->offset, key, size);
4997}
4998
4999static void
5000hist_trigger_actions(struct hist_trigger_data *hist_data,
5001 struct tracing_map_elt *elt,
5002 struct trace_buffer *buffer, void *rec,
5003 struct ring_buffer_event *rbe, void *key,
5004 u64 *var_ref_vals)
5005{
5006 struct action_data *data;
5007 unsigned int i;
5008
5009 for (i = 0; i < hist_data->n_actions; i++) {
5010 data = hist_data->actions[i];
5011 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5012 }
5013}
5014
5015static void event_hist_trigger(struct event_trigger_data *data,
5016 struct trace_buffer *buffer, void *rec,
5017 struct ring_buffer_event *rbe)
5018{
5019 struct hist_trigger_data *hist_data = data->private_data;
5020 bool use_compound_key = (hist_data->n_keys > 1);
5021 unsigned long entries[HIST_STACKTRACE_DEPTH];
5022 u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5023 char compound_key[HIST_KEY_SIZE_MAX];
5024 struct tracing_map_elt *elt = NULL;
5025 struct hist_field *key_field;
5026 u64 field_contents;
5027 void *key = NULL;
5028 unsigned int i;
5029
5030 memset(compound_key, 0, hist_data->key_size);
5031
5032 for_each_hist_key_field(i, hist_data) {
5033 key_field = hist_data->fields[i];
5034
5035 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5036 memset(entries, 0, HIST_STACKTRACE_SIZE);
5037 stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5038 HIST_STACKTRACE_SKIP);
5039 key = entries;
5040 } else {
5041 field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
5042 if (key_field->flags & HIST_FIELD_FL_STRING) {
5043 key = (void *)(unsigned long)field_contents;
5044 use_compound_key = true;
5045 } else
5046 key = (void *)&field_contents;
5047 }
5048
5049 if (use_compound_key)
5050 add_to_key(compound_key, key, key_field, rec);
5051 }
5052
5053 if (use_compound_key)
5054 key = compound_key;
5055
5056 if (hist_data->n_var_refs &&
5057 !resolve_var_refs(hist_data, key, var_ref_vals, false))
5058 return;
5059
5060 elt = tracing_map_insert(hist_data->map, key);
5061 if (!elt)
5062 return;
5063
5064 hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5065
5066 if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5067 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5068}
5069
5070static void hist_trigger_stacktrace_print(struct seq_file *m,
5071 unsigned long *stacktrace_entries,
5072 unsigned int max_entries)
5073{
5074 unsigned int spaces = 8;
5075 unsigned int i;
5076
5077 for (i = 0; i < max_entries; i++) {
5078 if (!stacktrace_entries[i])
5079 return;
5080
5081 seq_printf(m, "%*c", 1 + spaces, ' ');
5082 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5083 }
5084}
5085
5086static void hist_trigger_print_key(struct seq_file *m,
5087 struct hist_trigger_data *hist_data,
5088 void *key,
5089 struct tracing_map_elt *elt)
5090{
5091 struct hist_field *key_field;
5092 bool multiline = false;
5093 const char *field_name;
5094 unsigned int i;
5095 u64 uval;
5096
5097 seq_puts(m, "{ ");
5098
5099 for_each_hist_key_field(i, hist_data) {
5100 key_field = hist_data->fields[i];
5101
5102 if (i > hist_data->n_vals)
5103 seq_puts(m, ", ");
5104
5105 field_name = hist_field_name(key_field, 0);
5106
5107 if (key_field->flags & HIST_FIELD_FL_HEX) {
5108 uval = *(u64 *)(key + key_field->offset);
5109 seq_printf(m, "%s: %llx", field_name, uval);
5110 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5111 uval = *(u64 *)(key + key_field->offset);
5112 seq_printf(m, "%s: [%llx] %-45ps", field_name,
5113 uval, (void *)(uintptr_t)uval);
5114 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5115 uval = *(u64 *)(key + key_field->offset);
5116 seq_printf(m, "%s: [%llx] %-55pS", field_name,
5117 uval, (void *)(uintptr_t)uval);
5118 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5119 struct hist_elt_data *elt_data = elt->private_data;
5120 char *comm;
5121
5122 if (WARN_ON_ONCE(!elt_data))
5123 return;
5124
5125 comm = elt_data->comm;
5126
5127 uval = *(u64 *)(key + key_field->offset);
5128 seq_printf(m, "%s: %-16s[%10llu]", field_name,
5129 comm, uval);
5130 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5131 const char *syscall_name;
5132
5133 uval = *(u64 *)(key + key_field->offset);
5134 syscall_name = get_syscall_name(uval);
5135 if (!syscall_name)
5136 syscall_name = "unknown_syscall";
5137
5138 seq_printf(m, "%s: %-30s[%3llu]", field_name,
5139 syscall_name, uval);
5140 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5141 seq_puts(m, "stacktrace:\n");
5142 hist_trigger_stacktrace_print(m,
5143 key + key_field->offset,
5144 HIST_STACKTRACE_DEPTH);
5145 multiline = true;
5146 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5147 seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5148 *(u64 *)(key + key_field->offset));
5149 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5150 unsigned long buckets = key_field->buckets;
5151 uval = *(u64 *)(key + key_field->offset);
5152 seq_printf(m, "%s: ~ %llu-%llu", field_name,
5153 uval, uval + buckets -1);
5154 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5155 seq_printf(m, "%s: %-50s", field_name,
5156 (char *)(key + key_field->offset));
5157 } else {
5158 uval = *(u64 *)(key + key_field->offset);
5159 seq_printf(m, "%s: %10llu", field_name, uval);
5160 }
5161 }
5162
5163 if (!multiline)
5164 seq_puts(m, " ");
5165
5166 seq_puts(m, "}");
5167}
5168
5169static void hist_trigger_entry_print(struct seq_file *m,
5170 struct hist_trigger_data *hist_data,
5171 void *key,
5172 struct tracing_map_elt *elt)
5173{
5174 const char *field_name;
5175 unsigned int i;
5176
5177 hist_trigger_print_key(m, hist_data, key, elt);
5178
5179 seq_printf(m, " hitcount: %10llu",
5180 tracing_map_read_sum(elt, HITCOUNT_IDX));
5181
5182 for (i = 1; i < hist_data->n_vals; i++) {
5183 field_name = hist_field_name(hist_data->fields[i], 0);
5184
5185 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5186 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5187 continue;
5188
5189 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5190 seq_printf(m, " %s: %10llx", field_name,
5191 tracing_map_read_sum(elt, i));
5192 } else {
5193 seq_printf(m, " %s: %10llu", field_name,
5194 tracing_map_read_sum(elt, i));
5195 }
5196 }
5197
5198 print_actions(m, hist_data, elt);
5199
5200 seq_puts(m, "\n");
5201}
5202
5203static int print_entries(struct seq_file *m,
5204 struct hist_trigger_data *hist_data)
5205{
5206 struct tracing_map_sort_entry **sort_entries = NULL;
5207 struct tracing_map *map = hist_data->map;
5208 int i, n_entries;
5209
5210 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5211 hist_data->n_sort_keys,
5212 &sort_entries);
5213 if (n_entries < 0)
5214 return n_entries;
5215
5216 for (i = 0; i < n_entries; i++)
5217 hist_trigger_entry_print(m, hist_data,
5218 sort_entries[i]->key,
5219 sort_entries[i]->elt);
5220
5221 tracing_map_destroy_sort_entries(sort_entries, n_entries);
5222
5223 return n_entries;
5224}
5225
5226static void hist_trigger_show(struct seq_file *m,
5227 struct event_trigger_data *data, int n)
5228{
5229 struct hist_trigger_data *hist_data;
5230 int n_entries;
5231
5232 if (n > 0)
5233 seq_puts(m, "\n\n");
5234
5235 seq_puts(m, "# event histogram\n#\n# trigger info: ");
5236 data->ops->print(m, data->ops, data);
5237 seq_puts(m, "#\n\n");
5238
5239 hist_data = data->private_data;
5240 n_entries = print_entries(m, hist_data);
5241 if (n_entries < 0)
5242 n_entries = 0;
5243
5244 track_data_snapshot_print(m, hist_data);
5245
5246 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
5247 (u64)atomic64_read(&hist_data->map->hits),
5248 n_entries, (u64)atomic64_read(&hist_data->map->drops));
5249}
5250
5251static int hist_show(struct seq_file *m, void *v)
5252{
5253 struct event_trigger_data *data;
5254 struct trace_event_file *event_file;
5255 int n = 0, ret = 0;
5256
5257 mutex_lock(&event_mutex);
5258
5259 event_file = event_file_data(m->private);
5260 if (unlikely(!event_file)) {
5261 ret = -ENODEV;
5262 goto out_unlock;
5263 }
5264
5265 list_for_each_entry(data, &event_file->triggers, list) {
5266 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5267 hist_trigger_show(m, data, n++);
5268 }
5269
5270 out_unlock:
5271 mutex_unlock(&event_mutex);
5272
5273 return ret;
5274}
5275
5276static int event_hist_open(struct inode *inode, struct file *file)
5277{
5278 int ret;
5279
5280 ret = security_locked_down(LOCKDOWN_TRACEFS);
5281 if (ret)
5282 return ret;
5283
5284 return single_open(file, hist_show, file);
5285}
5286
5287const struct file_operations event_hist_fops = {
5288 .open = event_hist_open,
5289 .read = seq_read,
5290 .llseek = seq_lseek,
5291 .release = single_release,
5292};
5293
5294#ifdef CONFIG_HIST_TRIGGERS_DEBUG
5295static void hist_field_debug_show_flags(struct seq_file *m,
5296 unsigned long flags)
5297{
5298 seq_puts(m, " flags:\n");
5299
5300 if (flags & HIST_FIELD_FL_KEY)
5301 seq_puts(m, " HIST_FIELD_FL_KEY\n");
5302 else if (flags & HIST_FIELD_FL_HITCOUNT)
5303 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n");
5304 else if (flags & HIST_FIELD_FL_VAR)
5305 seq_puts(m, " HIST_FIELD_FL_VAR\n");
5306 else if (flags & HIST_FIELD_FL_VAR_REF)
5307 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n");
5308 else
5309 seq_puts(m, " VAL: normal u64 value\n");
5310
5311 if (flags & HIST_FIELD_FL_ALIAS)
5312 seq_puts(m, " HIST_FIELD_FL_ALIAS\n");
5313 else if (flags & HIST_FIELD_FL_CONST)
5314 seq_puts(m, " HIST_FIELD_FL_CONST\n");
5315}
5316
5317static int hist_field_debug_show(struct seq_file *m,
5318 struct hist_field *field, unsigned long flags)
5319{
5320 if ((field->flags & flags) != flags) {
5321 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5322 return -EINVAL;
5323 }
5324
5325 hist_field_debug_show_flags(m, field->flags);
5326 if (field->field)
5327 seq_printf(m, " ftrace_event_field name: %s\n",
5328 field->field->name);
5329
5330 if (field->flags & HIST_FIELD_FL_VAR) {
5331 seq_printf(m, " var.name: %s\n", field->var.name);
5332 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
5333 field->var.idx);
5334 }
5335
5336 if (field->flags & HIST_FIELD_FL_CONST)
5337 seq_printf(m, " constant: %llu\n", field->constant);
5338
5339 if (field->flags & HIST_FIELD_FL_ALIAS)
5340 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n",
5341 field->var_ref_idx);
5342
5343 if (field->flags & HIST_FIELD_FL_VAR_REF) {
5344 seq_printf(m, " name: %s\n", field->name);
5345 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
5346 field->var.idx);
5347 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data);
5348 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n",
5349 field->var_ref_idx);
5350 if (field->system)
5351 seq_printf(m, " system: %s\n", field->system);
5352 if (field->event_name)
5353 seq_printf(m, " event_name: %s\n", field->event_name);
5354 }
5355
5356 seq_printf(m, " type: %s\n", field->type);
5357 seq_printf(m, " size: %u\n", field->size);
5358 seq_printf(m, " is_signed: %u\n", field->is_signed);
5359
5360 return 0;
5361}
5362
5363static int field_var_debug_show(struct seq_file *m,
5364 struct field_var *field_var, unsigned int i,
5365 bool save_vars)
5366{
5367 const char *vars_name = save_vars ? "save_vars" : "field_vars";
5368 struct hist_field *field;
5369 int ret = 0;
5370
5371 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i);
5372
5373 field = field_var->var;
5374
5375 seq_printf(m, "\n %s[%d].var:\n", vars_name, i);
5376
5377 hist_field_debug_show_flags(m, field->flags);
5378 seq_printf(m, " var.name: %s\n", field->var.name);
5379 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
5380 field->var.idx);
5381
5382 field = field_var->val;
5383
5384 seq_printf(m, "\n %s[%d].val:\n", vars_name, i);
5385 if (field->field)
5386 seq_printf(m, " ftrace_event_field name: %s\n",
5387 field->field->name);
5388 else {
5389 ret = -EINVAL;
5390 goto out;
5391 }
5392
5393 seq_printf(m, " type: %s\n", field->type);
5394 seq_printf(m, " size: %u\n", field->size);
5395 seq_printf(m, " is_signed: %u\n", field->is_signed);
5396out:
5397 return ret;
5398}
5399
5400static int hist_action_debug_show(struct seq_file *m,
5401 struct action_data *data, int i)
5402{
5403 int ret = 0;
5404
5405 if (data->handler == HANDLER_ONMAX ||
5406 data->handler == HANDLER_ONCHANGE) {
5407 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i);
5408 ret = hist_field_debug_show(m, data->track_data.var_ref,
5409 HIST_FIELD_FL_VAR_REF);
5410 if (ret)
5411 goto out;
5412
5413 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i);
5414 ret = hist_field_debug_show(m, data->track_data.track_var,
5415 HIST_FIELD_FL_VAR);
5416 if (ret)
5417 goto out;
5418 }
5419
5420 if (data->handler == HANDLER_ONMATCH) {
5421 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n",
5422 i, data->match_data.event_system);
5423 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n",
5424 i, data->match_data.event);
5425 }
5426out:
5427 return ret;
5428}
5429
5430static int hist_actions_debug_show(struct seq_file *m,
5431 struct hist_trigger_data *hist_data)
5432{
5433 int i, ret = 0;
5434
5435 if (hist_data->n_actions)
5436 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n");
5437
5438 for (i = 0; i < hist_data->n_actions; i++) {
5439 struct action_data *action = hist_data->actions[i];
5440
5441 ret = hist_action_debug_show(m, action, i);
5442 if (ret)
5443 goto out;
5444 }
5445
5446 if (hist_data->n_save_vars)
5447 seq_puts(m, "\n save action variables (save() params):\n");
5448
5449 for (i = 0; i < hist_data->n_save_vars; i++) {
5450 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5451 if (ret)
5452 goto out;
5453 }
5454out:
5455 return ret;
5456}
5457
5458static void hist_trigger_debug_show(struct seq_file *m,
5459 struct event_trigger_data *data, int n)
5460{
5461 struct hist_trigger_data *hist_data;
5462 int i, ret;
5463
5464 if (n > 0)
5465 seq_puts(m, "\n\n");
5466
5467 seq_puts(m, "# event histogram\n#\n# trigger info: ");
5468 data->ops->print(m, data->ops, data);
5469 seq_puts(m, "#\n\n");
5470
5471 hist_data = data->private_data;
5472
5473 seq_printf(m, "hist_data: %p\n\n", hist_data);
5474 seq_printf(m, " n_vals: %u\n", hist_data->n_vals);
5475 seq_printf(m, " n_keys: %u\n", hist_data->n_keys);
5476 seq_printf(m, " n_fields: %u\n", hist_data->n_fields);
5477
5478 seq_puts(m, "\n val fields:\n\n");
5479
5480 seq_puts(m, " hist_data->fields[0]:\n");
5481 ret = hist_field_debug_show(m, hist_data->fields[0],
5482 HIST_FIELD_FL_HITCOUNT);
5483 if (ret)
5484 return;
5485
5486 for (i = 1; i < hist_data->n_vals; i++) {
5487 seq_printf(m, "\n hist_data->fields[%d]:\n", i);
5488 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5489 if (ret)
5490 return;
5491 }
5492
5493 seq_puts(m, "\n key fields:\n");
5494
5495 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5496 seq_printf(m, "\n hist_data->fields[%d]:\n", i);
5497 ret = hist_field_debug_show(m, hist_data->fields[i],
5498 HIST_FIELD_FL_KEY);
5499 if (ret)
5500 return;
5501 }
5502
5503 if (hist_data->n_var_refs)
5504 seq_puts(m, "\n variable reference fields:\n");
5505
5506 for (i = 0; i < hist_data->n_var_refs; i++) {
5507 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i);
5508 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5509 HIST_FIELD_FL_VAR_REF);
5510 if (ret)
5511 return;
5512 }
5513
5514 if (hist_data->n_field_vars)
5515 seq_puts(m, "\n field variables:\n");
5516
5517 for (i = 0; i < hist_data->n_field_vars; i++) {
5518 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5519 if (ret)
5520 return;
5521 }
5522
5523 ret = hist_actions_debug_show(m, hist_data);
5524 if (ret)
5525 return;
5526}
5527
5528static int hist_debug_show(struct seq_file *m, void *v)
5529{
5530 struct event_trigger_data *data;
5531 struct trace_event_file *event_file;
5532 int n = 0, ret = 0;
5533
5534 mutex_lock(&event_mutex);
5535
5536 event_file = event_file_data(m->private);
5537 if (unlikely(!event_file)) {
5538 ret = -ENODEV;
5539 goto out_unlock;
5540 }
5541
5542 list_for_each_entry(data, &event_file->triggers, list) {
5543 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5544 hist_trigger_debug_show(m, data, n++);
5545 }
5546
5547 out_unlock:
5548 mutex_unlock(&event_mutex);
5549
5550 return ret;
5551}
5552
5553static int event_hist_debug_open(struct inode *inode, struct file *file)
5554{
5555 int ret;
5556
5557 ret = security_locked_down(LOCKDOWN_TRACEFS);
5558 if (ret)
5559 return ret;
5560
5561 return single_open(file, hist_debug_show, file);
5562}
5563
5564const struct file_operations event_hist_debug_fops = {
5565 .open = event_hist_debug_open,
5566 .read = seq_read,
5567 .llseek = seq_lseek,
5568 .release = single_release,
5569};
5570#endif
5571
5572static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5573{
5574 const char *field_name = hist_field_name(hist_field, 0);
5575
5576 if (hist_field->var.name)
5577 seq_printf(m, "%s=", hist_field->var.name);
5578
5579 if (hist_field->flags & HIST_FIELD_FL_CPU)
5580 seq_puts(m, "common_cpu");
5581 else if (hist_field->flags & HIST_FIELD_FL_CONST)
5582 seq_printf(m, "%llu", hist_field->constant);
5583 else if (field_name) {
5584 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5585 hist_field->flags & HIST_FIELD_FL_ALIAS)
5586 seq_putc(m, '$');
5587 seq_printf(m, "%s", field_name);
5588 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5589 seq_puts(m, "common_timestamp");
5590
5591 if (hist_field->flags) {
5592 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5593 !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5594 const char *flags = get_hist_field_flags(hist_field);
5595
5596 if (flags)
5597 seq_printf(m, ".%s", flags);
5598 }
5599 }
5600 if (hist_field->buckets)
5601 seq_printf(m, "=%ld", hist_field->buckets);
5602}
5603
5604static int event_hist_trigger_print(struct seq_file *m,
5605 struct event_trigger_ops *ops,
5606 struct event_trigger_data *data)
5607{
5608 struct hist_trigger_data *hist_data = data->private_data;
5609 struct hist_field *field;
5610 bool have_var = false;
5611 unsigned int i;
5612
5613 seq_puts(m, "hist:");
5614
5615 if (data->name)
5616 seq_printf(m, "%s:", data->name);
5617
5618 seq_puts(m, "keys=");
5619
5620 for_each_hist_key_field(i, hist_data) {
5621 field = hist_data->fields[i];
5622
5623 if (i > hist_data->n_vals)
5624 seq_puts(m, ",");
5625
5626 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5627 seq_puts(m, "stacktrace");
5628 else
5629 hist_field_print(m, field);
5630 }
5631
5632 seq_puts(m, ":vals=");
5633
5634 for_each_hist_val_field(i, hist_data) {
5635 field = hist_data->fields[i];
5636 if (field->flags & HIST_FIELD_FL_VAR) {
5637 have_var = true;
5638 continue;
5639 }
5640
5641 if (i == HITCOUNT_IDX)
5642 seq_puts(m, "hitcount");
5643 else {
5644 seq_puts(m, ",");
5645 hist_field_print(m, field);
5646 }
5647 }
5648
5649 if (have_var) {
5650 unsigned int n = 0;
5651
5652 seq_puts(m, ":");
5653
5654 for_each_hist_val_field(i, hist_data) {
5655 field = hist_data->fields[i];
5656
5657 if (field->flags & HIST_FIELD_FL_VAR) {
5658 if (n++)
5659 seq_puts(m, ",");
5660 hist_field_print(m, field);
5661 }
5662 }
5663 }
5664
5665 seq_puts(m, ":sort=");
5666
5667 for (i = 0; i < hist_data->n_sort_keys; i++) {
5668 struct tracing_map_sort_key *sort_key;
5669 unsigned int idx, first_key_idx;
5670
5671
5672 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5673
5674 sort_key = &hist_data->sort_keys[i];
5675 idx = sort_key->field_idx;
5676
5677 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5678 return -EINVAL;
5679
5680 if (i > 0)
5681 seq_puts(m, ",");
5682
5683 if (idx == HITCOUNT_IDX)
5684 seq_puts(m, "hitcount");
5685 else {
5686 if (idx >= first_key_idx)
5687 idx += hist_data->n_vars;
5688 hist_field_print(m, hist_data->fields[idx]);
5689 }
5690
5691 if (sort_key->descending)
5692 seq_puts(m, ".descending");
5693 }
5694 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5695 if (hist_data->enable_timestamps)
5696 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5697
5698 print_actions_spec(m, hist_data);
5699
5700 if (data->filter_str)
5701 seq_printf(m, " if %s", data->filter_str);
5702
5703 if (data->paused)
5704 seq_puts(m, " [paused]");
5705 else
5706 seq_puts(m, " [active]");
5707
5708 seq_putc(m, '\n');
5709
5710 return 0;
5711}
5712
5713static int event_hist_trigger_init(struct event_trigger_ops *ops,
5714 struct event_trigger_data *data)
5715{
5716 struct hist_trigger_data *hist_data = data->private_data;
5717
5718 if (!data->ref && hist_data->attrs->name)
5719 save_named_trigger(hist_data->attrs->name, data);
5720
5721 data->ref++;
5722
5723 return 0;
5724}
5725
5726static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5727{
5728 struct trace_event_file *file;
5729 unsigned int i;
5730 char *cmd;
5731 int ret;
5732
5733 for (i = 0; i < hist_data->n_field_var_hists; i++) {
5734 file = hist_data->field_var_hists[i]->hist_data->event_file;
5735 cmd = hist_data->field_var_hists[i]->cmd;
5736 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
5737 "!hist", "hist", cmd);
5738 WARN_ON_ONCE(ret < 0);
5739 }
5740}
5741
5742static void event_hist_trigger_free(struct event_trigger_ops *ops,
5743 struct event_trigger_data *data)
5744{
5745 struct hist_trigger_data *hist_data = data->private_data;
5746
5747 if (WARN_ON_ONCE(data->ref <= 0))
5748 return;
5749
5750 data->ref--;
5751 if (!data->ref) {
5752 if (data->name)
5753 del_named_trigger(data);
5754
5755 trigger_data_free(data);
5756
5757 remove_hist_vars(hist_data);
5758
5759 unregister_field_var_hists(hist_data);
5760
5761 destroy_hist_data(hist_data);
5762 }
5763}
5764
5765static struct event_trigger_ops event_hist_trigger_ops = {
5766 .trigger = event_hist_trigger,
5767 .print = event_hist_trigger_print,
5768 .init = event_hist_trigger_init,
5769 .free = event_hist_trigger_free,
5770};
5771
5772static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5773 struct event_trigger_data *data)
5774{
5775 data->ref++;
5776
5777 save_named_trigger(data->named_data->name, data);
5778
5779 event_hist_trigger_init(ops, data->named_data);
5780
5781 return 0;
5782}
5783
5784static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5785 struct event_trigger_data *data)
5786{
5787 if (WARN_ON_ONCE(data->ref <= 0))
5788 return;
5789
5790 event_hist_trigger_free(ops, data->named_data);
5791
5792 data->ref--;
5793 if (!data->ref) {
5794 del_named_trigger(data);
5795 trigger_data_free(data);
5796 }
5797}
5798
5799static struct event_trigger_ops event_hist_trigger_named_ops = {
5800 .trigger = event_hist_trigger,
5801 .print = event_hist_trigger_print,
5802 .init = event_hist_trigger_named_init,
5803 .free = event_hist_trigger_named_free,
5804};
5805
5806static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5807 char *param)
5808{
5809 return &event_hist_trigger_ops;
5810}
5811
5812static void hist_clear(struct event_trigger_data *data)
5813{
5814 struct hist_trigger_data *hist_data = data->private_data;
5815
5816 if (data->name)
5817 pause_named_trigger(data);
5818
5819 tracepoint_synchronize_unregister();
5820
5821 tracing_map_clear(hist_data->map);
5822
5823 if (data->name)
5824 unpause_named_trigger(data);
5825}
5826
5827static bool compatible_field(struct ftrace_event_field *field,
5828 struct ftrace_event_field *test_field)
5829{
5830 if (field == test_field)
5831 return true;
5832 if (field == NULL || test_field == NULL)
5833 return false;
5834 if (strcmp(field->name, test_field->name) != 0)
5835 return false;
5836 if (strcmp(field->type, test_field->type) != 0)
5837 return false;
5838 if (field->size != test_field->size)
5839 return false;
5840 if (field->is_signed != test_field->is_signed)
5841 return false;
5842
5843 return true;
5844}
5845
5846static bool hist_trigger_match(struct event_trigger_data *data,
5847 struct event_trigger_data *data_test,
5848 struct event_trigger_data *named_data,
5849 bool ignore_filter)
5850{
5851 struct tracing_map_sort_key *sort_key, *sort_key_test;
5852 struct hist_trigger_data *hist_data, *hist_data_test;
5853 struct hist_field *key_field, *key_field_test;
5854 unsigned int i;
5855
5856 if (named_data && (named_data != data_test) &&
5857 (named_data != data_test->named_data))
5858 return false;
5859
5860 if (!named_data && is_named_trigger(data_test))
5861 return false;
5862
5863 hist_data = data->private_data;
5864 hist_data_test = data_test->private_data;
5865
5866 if (hist_data->n_vals != hist_data_test->n_vals ||
5867 hist_data->n_fields != hist_data_test->n_fields ||
5868 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5869 return false;
5870
5871 if (!ignore_filter) {
5872 if ((data->filter_str && !data_test->filter_str) ||
5873 (!data->filter_str && data_test->filter_str))
5874 return false;
5875 }
5876
5877 for_each_hist_field(i, hist_data) {
5878 key_field = hist_data->fields[i];
5879 key_field_test = hist_data_test->fields[i];
5880
5881 if (key_field->flags != key_field_test->flags)
5882 return false;
5883 if (!compatible_field(key_field->field, key_field_test->field))
5884 return false;
5885 if (key_field->offset != key_field_test->offset)
5886 return false;
5887 if (key_field->size != key_field_test->size)
5888 return false;
5889 if (key_field->is_signed != key_field_test->is_signed)
5890 return false;
5891 if (!!key_field->var.name != !!key_field_test->var.name)
5892 return false;
5893 if (key_field->var.name &&
5894 strcmp(key_field->var.name, key_field_test->var.name) != 0)
5895 return false;
5896 }
5897
5898 for (i = 0; i < hist_data->n_sort_keys; i++) {
5899 sort_key = &hist_data->sort_keys[i];
5900 sort_key_test = &hist_data_test->sort_keys[i];
5901
5902 if (sort_key->field_idx != sort_key_test->field_idx ||
5903 sort_key->descending != sort_key_test->descending)
5904 return false;
5905 }
5906
5907 if (!ignore_filter && data->filter_str &&
5908 (strcmp(data->filter_str, data_test->filter_str) != 0))
5909 return false;
5910
5911 if (!actions_match(hist_data, hist_data_test))
5912 return false;
5913
5914 return true;
5915}
5916
5917static int hist_register_trigger(char *glob,
5918 struct event_trigger_data *data,
5919 struct trace_event_file *file)
5920{
5921 struct hist_trigger_data *hist_data = data->private_data;
5922 struct event_trigger_data *test, *named_data = NULL;
5923 struct trace_array *tr = file->tr;
5924 int ret = 0;
5925
5926 if (hist_data->attrs->name) {
5927 named_data = find_named_trigger(hist_data->attrs->name);
5928 if (named_data) {
5929 if (!hist_trigger_match(data, named_data, named_data,
5930 true)) {
5931 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5932 ret = -EINVAL;
5933 goto out;
5934 }
5935 }
5936 }
5937
5938 if (hist_data->attrs->name && !named_data)
5939 goto new;
5940
5941 lockdep_assert_held(&event_mutex);
5942
5943 list_for_each_entry(test, &file->triggers, list) {
5944 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5945 if (!hist_trigger_match(data, test, named_data, false))
5946 continue;
5947 if (hist_data->attrs->pause)
5948 test->paused = true;
5949 else if (hist_data->attrs->cont)
5950 test->paused = false;
5951 else if (hist_data->attrs->clear)
5952 hist_clear(test);
5953 else {
5954 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5955 ret = -EEXIST;
5956 }
5957 goto out;
5958 }
5959 }
5960 new:
5961 if (hist_data->attrs->cont || hist_data->attrs->clear) {
5962 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5963 ret = -ENOENT;
5964 goto out;
5965 }
5966
5967 if (hist_data->attrs->pause)
5968 data->paused = true;
5969
5970 if (named_data) {
5971 data->private_data = named_data->private_data;
5972 set_named_trigger_data(data, named_data);
5973 data->ops = &event_hist_trigger_named_ops;
5974 }
5975
5976 if (data->ops->init) {
5977 ret = data->ops->init(data->ops, data);
5978 if (ret < 0)
5979 goto out;
5980 }
5981
5982 if (hist_data->enable_timestamps) {
5983 char *clock = hist_data->attrs->clock;
5984
5985 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5986 if (ret) {
5987 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5988 goto out;
5989 }
5990
5991 tracing_set_filter_buffering(file->tr, true);
5992 }
5993
5994 if (named_data)
5995 destroy_hist_data(hist_data);
5996
5997 ret++;
5998 out:
5999 return ret;
6000}
6001
6002static int hist_trigger_enable(struct event_trigger_data *data,
6003 struct trace_event_file *file)
6004{
6005 int ret = 0;
6006
6007 list_add_tail_rcu(&data->list, &file->triggers);
6008
6009 update_cond_flag(file);
6010
6011 if (trace_event_trigger_enable_disable(file, 1) < 0) {
6012 list_del_rcu(&data->list);
6013 update_cond_flag(file);
6014 ret--;
6015 }
6016
6017 return ret;
6018}
6019
6020static bool have_hist_trigger_match(struct event_trigger_data *data,
6021 struct trace_event_file *file)
6022{
6023 struct hist_trigger_data *hist_data = data->private_data;
6024 struct event_trigger_data *test, *named_data = NULL;
6025 bool match = false;
6026
6027 lockdep_assert_held(&event_mutex);
6028
6029 if (hist_data->attrs->name)
6030 named_data = find_named_trigger(hist_data->attrs->name);
6031
6032 list_for_each_entry(test, &file->triggers, list) {
6033 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6034 if (hist_trigger_match(data, test, named_data, false)) {
6035 match = true;
6036 break;
6037 }
6038 }
6039 }
6040
6041 return match;
6042}
6043
6044static bool hist_trigger_check_refs(struct event_trigger_data *data,
6045 struct trace_event_file *file)
6046{
6047 struct hist_trigger_data *hist_data = data->private_data;
6048 struct event_trigger_data *test, *named_data = NULL;
6049
6050 lockdep_assert_held(&event_mutex);
6051
6052 if (hist_data->attrs->name)
6053 named_data = find_named_trigger(hist_data->attrs->name);
6054
6055 list_for_each_entry(test, &file->triggers, list) {
6056 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6057 if (!hist_trigger_match(data, test, named_data, false))
6058 continue;
6059 hist_data = test->private_data;
6060 if (check_var_refs(hist_data))
6061 return true;
6062 break;
6063 }
6064 }
6065
6066 return false;
6067}
6068
6069static void hist_unregister_trigger(char *glob,
6070 struct event_trigger_data *data,
6071 struct trace_event_file *file)
6072{
6073 struct hist_trigger_data *hist_data = data->private_data;
6074 struct event_trigger_data *test, *named_data = NULL;
6075 bool unregistered = false;
6076
6077 lockdep_assert_held(&event_mutex);
6078
6079 if (hist_data->attrs->name)
6080 named_data = find_named_trigger(hist_data->attrs->name);
6081
6082 list_for_each_entry(test, &file->triggers, list) {
6083 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6084 if (!hist_trigger_match(data, test, named_data, false))
6085 continue;
6086 unregistered = true;
6087 list_del_rcu(&test->list);
6088 trace_event_trigger_enable_disable(file, 0);
6089 update_cond_flag(file);
6090 break;
6091 }
6092 }
6093
6094 if (unregistered && test->ops->free)
6095 test->ops->free(test->ops, test);
6096
6097 if (hist_data->enable_timestamps) {
6098 if (!hist_data->remove || unregistered)
6099 tracing_set_filter_buffering(file->tr, false);
6100 }
6101}
6102
6103static bool hist_file_check_refs(struct trace_event_file *file)
6104{
6105 struct hist_trigger_data *hist_data;
6106 struct event_trigger_data *test;
6107
6108 lockdep_assert_held(&event_mutex);
6109
6110 list_for_each_entry(test, &file->triggers, list) {
6111 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6112 hist_data = test->private_data;
6113 if (check_var_refs(hist_data))
6114 return true;
6115 }
6116 }
6117
6118 return false;
6119}
6120
6121static void hist_unreg_all(struct trace_event_file *file)
6122{
6123 struct event_trigger_data *test, *n;
6124 struct hist_trigger_data *hist_data;
6125 struct synth_event *se;
6126 const char *se_name;
6127
6128 lockdep_assert_held(&event_mutex);
6129
6130 if (hist_file_check_refs(file))
6131 return;
6132
6133 list_for_each_entry_safe(test, n, &file->triggers, list) {
6134 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6135 hist_data = test->private_data;
6136 list_del_rcu(&test->list);
6137 trace_event_trigger_enable_disable(file, 0);
6138
6139 se_name = trace_event_name(file->event_call);
6140 se = find_synth_event(se_name);
6141 if (se)
6142 se->ref--;
6143
6144 update_cond_flag(file);
6145 if (hist_data->enable_timestamps)
6146 tracing_set_filter_buffering(file->tr, false);
6147 if (test->ops->free)
6148 test->ops->free(test->ops, test);
6149 }
6150 }
6151}
6152
6153static int event_hist_trigger_parse(struct event_command *cmd_ops,
6154 struct trace_event_file *file,
6155 char *glob, char *cmd, char *param)
6156{
6157 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6158 struct event_trigger_data *trigger_data;
6159 struct hist_trigger_attrs *attrs;
6160 struct event_trigger_ops *trigger_ops;
6161 struct hist_trigger_data *hist_data;
6162 struct synth_event *se;
6163 const char *se_name;
6164 bool remove = false;
6165 char *trigger, *p, *start;
6166 int ret = 0;
6167
6168 lockdep_assert_held(&event_mutex);
6169
6170 WARN_ON(!glob);
6171
6172 if (strlen(glob)) {
6173 hist_err_clear();
6174 last_cmd_set(file, param);
6175 }
6176
6177 if (!param)
6178 return -EINVAL;
6179
6180 if (glob[0] == '!')
6181 remove = true;
6182
6183
6184
6185
6186
6187 p = trigger = param;
6188 do {
6189 p = strstr(p, "if");
6190 if (!p)
6191 break;
6192 if (p == param)
6193 return -EINVAL;
6194 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6195 p++;
6196 continue;
6197 }
6198 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6199 return -EINVAL;
6200 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6201 p++;
6202 continue;
6203 }
6204 break;
6205 } while (1);
6206
6207 if (!p)
6208 param = NULL;
6209 else {
6210 *(p - 1) = '\0';
6211 param = strstrip(p);
6212 trigger = strstrip(trigger);
6213 }
6214
6215
6216
6217
6218
6219 start = strstr(trigger, ".sym-offset");
6220 while (start) {
6221 *(start + 4) = 'X';
6222 start = strstr(start + 11, ".sym-offset");
6223 }
6224
6225 attrs = parse_hist_trigger_attrs(file->tr, trigger);
6226 if (IS_ERR(attrs))
6227 return PTR_ERR(attrs);
6228
6229 if (attrs->map_bits)
6230 hist_trigger_bits = attrs->map_bits;
6231
6232 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6233 if (IS_ERR(hist_data)) {
6234 destroy_hist_trigger_attrs(attrs);
6235 return PTR_ERR(hist_data);
6236 }
6237
6238 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6239
6240 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6241 if (!trigger_data) {
6242 ret = -ENOMEM;
6243 goto out_free;
6244 }
6245
6246 trigger_data->count = -1;
6247 trigger_data->ops = trigger_ops;
6248 trigger_data->cmd_ops = cmd_ops;
6249
6250 INIT_LIST_HEAD(&trigger_data->list);
6251 RCU_INIT_POINTER(trigger_data->filter, NULL);
6252
6253 trigger_data->private_data = hist_data;
6254
6255
6256 if (param && cmd_ops->set_filter) {
6257 ret = cmd_ops->set_filter(param, trigger_data, file);
6258 if (ret < 0)
6259 goto out_free;
6260 }
6261
6262 if (remove) {
6263 if (!have_hist_trigger_match(trigger_data, file))
6264 goto out_free;
6265
6266 if (hist_trigger_check_refs(trigger_data, file)) {
6267 ret = -EBUSY;
6268 goto out_free;
6269 }
6270
6271 cmd_ops->unreg(glob+1, trigger_data, file);
6272 se_name = trace_event_name(file->event_call);
6273 se = find_synth_event(se_name);
6274 if (se)
6275 se->ref--;
6276 ret = 0;
6277 goto out_free;
6278 }
6279
6280 ret = cmd_ops->reg(glob, trigger_data, file);
6281
6282
6283
6284
6285
6286 if (!ret) {
6287 if (!(attrs->pause || attrs->cont || attrs->clear))
6288 ret = -ENOENT;
6289 goto out_free;
6290 } else if (ret < 0)
6291 goto out_free;
6292
6293 if (get_named_trigger_data(trigger_data))
6294 goto enable;
6295
6296 if (has_hist_vars(hist_data))
6297 save_hist_vars(hist_data);
6298
6299 ret = create_actions(hist_data);
6300 if (ret)
6301 goto out_unreg;
6302
6303 ret = tracing_map_init(hist_data->map);
6304 if (ret)
6305 goto out_unreg;
6306enable:
6307 ret = hist_trigger_enable(trigger_data, file);
6308 if (ret)
6309 goto out_unreg;
6310
6311 se_name = trace_event_name(file->event_call);
6312 se = find_synth_event(se_name);
6313 if (se)
6314 se->ref++;
6315
6316 ret = 0;
6317 out:
6318 if (ret == 0)
6319 hist_err_clear();
6320
6321 return ret;
6322 out_unreg:
6323 cmd_ops->unreg(glob+1, trigger_data, file);
6324 out_free:
6325 if (cmd_ops->set_filter)
6326 cmd_ops->set_filter(NULL, trigger_data, NULL);
6327
6328 remove_hist_vars(hist_data);
6329
6330 kfree(trigger_data);
6331
6332 destroy_hist_data(hist_data);
6333 goto out;
6334}
6335
6336static struct event_command trigger_hist_cmd = {
6337 .name = "hist",
6338 .trigger_type = ETT_EVENT_HIST,
6339 .flags = EVENT_CMD_FL_NEEDS_REC,
6340 .parse = event_hist_trigger_parse,
6341 .reg = hist_register_trigger,
6342 .unreg = hist_unregister_trigger,
6343 .unreg_all = hist_unreg_all,
6344 .get_trigger_ops = event_hist_get_trigger_ops,
6345 .set_filter = set_trigger_filter,
6346};
6347
6348__init int register_trigger_hist_cmd(void)
6349{
6350 int ret;
6351
6352 ret = register_event_command(&trigger_hist_cmd);
6353 WARN_ON(ret < 0);
6354
6355 return ret;
6356}
6357
6358static void
6359hist_enable_trigger(struct event_trigger_data *data,
6360 struct trace_buffer *buffer, void *rec,
6361 struct ring_buffer_event *event)
6362{
6363 struct enable_trigger_data *enable_data = data->private_data;
6364 struct event_trigger_data *test;
6365
6366 list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6367 lockdep_is_held(&event_mutex)) {
6368 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6369 if (enable_data->enable)
6370 test->paused = false;
6371 else
6372 test->paused = true;
6373 }
6374 }
6375}
6376
6377static void
6378hist_enable_count_trigger(struct event_trigger_data *data,
6379 struct trace_buffer *buffer, void *rec,
6380 struct ring_buffer_event *event)
6381{
6382 if (!data->count)
6383 return;
6384
6385 if (data->count != -1)
6386 (data->count)--;
6387
6388 hist_enable_trigger(data, buffer, rec, event);
6389}
6390
6391static struct event_trigger_ops hist_enable_trigger_ops = {
6392 .trigger = hist_enable_trigger,
6393 .print = event_enable_trigger_print,
6394 .init = event_trigger_init,
6395 .free = event_enable_trigger_free,
6396};
6397
6398static struct event_trigger_ops hist_enable_count_trigger_ops = {
6399 .trigger = hist_enable_count_trigger,
6400 .print = event_enable_trigger_print,
6401 .init = event_trigger_init,
6402 .free = event_enable_trigger_free,
6403};
6404
6405static struct event_trigger_ops hist_disable_trigger_ops = {
6406 .trigger = hist_enable_trigger,
6407 .print = event_enable_trigger_print,
6408 .init = event_trigger_init,
6409 .free = event_enable_trigger_free,
6410};
6411
6412static struct event_trigger_ops hist_disable_count_trigger_ops = {
6413 .trigger = hist_enable_count_trigger,
6414 .print = event_enable_trigger_print,
6415 .init = event_trigger_init,
6416 .free = event_enable_trigger_free,
6417};
6418
6419static struct event_trigger_ops *
6420hist_enable_get_trigger_ops(char *cmd, char *param)
6421{
6422 struct event_trigger_ops *ops;
6423 bool enable;
6424
6425 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6426
6427 if (enable)
6428 ops = param ? &hist_enable_count_trigger_ops :
6429 &hist_enable_trigger_ops;
6430 else
6431 ops = param ? &hist_disable_count_trigger_ops :
6432 &hist_disable_trigger_ops;
6433
6434 return ops;
6435}
6436
6437static void hist_enable_unreg_all(struct trace_event_file *file)
6438{
6439 struct event_trigger_data *test, *n;
6440
6441 list_for_each_entry_safe(test, n, &file->triggers, list) {
6442 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6443 list_del_rcu(&test->list);
6444 update_cond_flag(file);
6445 trace_event_trigger_enable_disable(file, 0);
6446 if (test->ops->free)
6447 test->ops->free(test->ops, test);
6448 }
6449 }
6450}
6451
6452static struct event_command trigger_hist_enable_cmd = {
6453 .name = ENABLE_HIST_STR,
6454 .trigger_type = ETT_HIST_ENABLE,
6455 .parse = event_enable_trigger_parse,
6456 .reg = event_enable_register_trigger,
6457 .unreg = event_enable_unregister_trigger,
6458 .unreg_all = hist_enable_unreg_all,
6459 .get_trigger_ops = hist_enable_get_trigger_ops,
6460 .set_filter = set_trigger_filter,
6461};
6462
6463static struct event_command trigger_hist_disable_cmd = {
6464 .name = DISABLE_HIST_STR,
6465 .trigger_type = ETT_HIST_ENABLE,
6466 .parse = event_enable_trigger_parse,
6467 .reg = event_enable_register_trigger,
6468 .unreg = event_enable_unregister_trigger,
6469 .unreg_all = hist_enable_unreg_all,
6470 .get_trigger_ops = hist_enable_get_trigger_ops,
6471 .set_filter = set_trigger_filter,
6472};
6473
6474static __init void unregister_trigger_hist_enable_disable_cmds(void)
6475{
6476 unregister_event_command(&trigger_hist_enable_cmd);
6477 unregister_event_command(&trigger_hist_disable_cmd);
6478}
6479
6480__init int register_trigger_hist_enable_disable_cmds(void)
6481{
6482 int ret;
6483
6484 ret = register_event_command(&trigger_hist_enable_cmd);
6485 if (WARN_ON(ret < 0))
6486 return ret;
6487 ret = register_event_command(&trigger_hist_disable_cmd);
6488 if (WARN_ON(ret < 0))
6489 unregister_trigger_hist_enable_disable_cmds();
6490
6491 return ret;
6492}
6493