1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/module.h>
21#include <linux/uaccess.h>
22
23#include "trace_probe.h"
24
25#define KPROBE_EVENT_SYSTEM "kprobes"
26
27
28
29
30struct trace_kprobe {
31 struct list_head list;
32 struct kretprobe rp;
33 unsigned long nhit;
34 const char *symbol;
35 struct trace_probe tp;
36};
37
38struct event_file_link {
39 struct ftrace_event_file *file;
40 struct list_head list;
41};
42
43#define SIZEOF_TRACE_KPROBE(n) \
44 (offsetof(struct trace_kprobe, tp.args) + \
45 (sizeof(struct probe_arg) * (n)))
46
47
48static __kprobes bool trace_kprobe_is_return(struct trace_kprobe *tk)
49{
50 return tk->rp.handler != NULL;
51}
52
53static __kprobes const char *trace_kprobe_symbol(struct trace_kprobe *tk)
54{
55 return tk->symbol ? tk->symbol : "unknown";
56}
57
58static __kprobes unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
59{
60 return tk->rp.kp.offset;
61}
62
63static __kprobes bool trace_kprobe_has_gone(struct trace_kprobe *tk)
64{
65 return !!(kprobe_gone(&tk->rp.kp));
66}
67
68static __kprobes bool trace_kprobe_within_module(struct trace_kprobe *tk,
69 struct module *mod)
70{
71 int len = strlen(mod->name);
72 const char *name = trace_kprobe_symbol(tk);
73 return strncmp(mod->name, name, len) == 0 && name[len] == ':';
74}
75
76static __kprobes bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
77{
78 return !!strchr(trace_kprobe_symbol(tk), ':');
79}
80
81static int register_kprobe_event(struct trace_kprobe *tk);
82static int unregister_kprobe_event(struct trace_kprobe *tk);
83
84static DEFINE_MUTEX(probe_lock);
85static LIST_HEAD(probe_list);
86
87static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
88static int kretprobe_dispatcher(struct kretprobe_instance *ri,
89 struct pt_regs *regs);
90
91
92struct symbol_cache {
93 char *symbol;
94 long offset;
95 unsigned long addr;
96};
97
98unsigned long update_symbol_cache(struct symbol_cache *sc)
99{
100 sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
101
102 if (sc->addr)
103 sc->addr += sc->offset;
104
105 return sc->addr;
106}
107
108void free_symbol_cache(struct symbol_cache *sc)
109{
110 kfree(sc->symbol);
111 kfree(sc);
112}
113
114struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
115{
116 struct symbol_cache *sc;
117
118 if (!sym || strlen(sym) == 0)
119 return NULL;
120
121 sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
122 if (!sc)
123 return NULL;
124
125 sc->symbol = kstrdup(sym, GFP_KERNEL);
126 if (!sc->symbol) {
127 kfree(sc);
128 return NULL;
129 }
130 sc->offset = offset;
131 update_symbol_cache(sc);
132
133 return sc;
134}
135
136
137
138
139#define DEFINE_FETCH_stack(type) \
140static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\
141 void *offset, void *dest) \
142{ \
143 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
144 (unsigned int)((unsigned long)offset)); \
145}
146DEFINE_BASIC_FETCH_FUNCS(stack)
147
148#define fetch_stack_string NULL
149#define fetch_stack_string_size NULL
150
151#define DEFINE_FETCH_memory(type) \
152static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
153 void *addr, void *dest) \
154{ \
155 type retval; \
156 if (probe_kernel_address(addr, retval)) \
157 *(type *)dest = 0; \
158 else \
159 *(type *)dest = retval; \
160}
161DEFINE_BASIC_FETCH_FUNCS(memory)
162
163
164
165
166static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
167 void *addr, void *dest)
168{
169 int maxlen = get_rloc_len(*(u32 *)dest);
170 u8 *dst = get_rloc_data(dest);
171 long ret;
172
173 if (!maxlen)
174 return;
175
176
177
178
179
180 ret = strncpy_from_unsafe(dst, addr, maxlen);
181
182 if (ret < 0) {
183 dst[0] = '\0';
184 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
185 } else {
186 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
187 }
188}
189
190
191static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
192 void *addr, void *dest)
193{
194 mm_segment_t old_fs;
195 int ret, len = 0;
196 u8 c;
197
198 old_fs = get_fs();
199 set_fs(KERNEL_DS);
200 pagefault_disable();
201
202 do {
203 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
204 len++;
205 } while (c && ret == 0 && len < MAX_STRING_SIZE);
206
207 pagefault_enable();
208 set_fs(old_fs);
209
210 if (ret < 0)
211 *(u32 *)dest = 0;
212 else
213 *(u32 *)dest = len;
214}
215
216#define DEFINE_FETCH_symbol(type) \
217__kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, \
218 void *data, void *dest) \
219{ \
220 struct symbol_cache *sc = data; \
221 if (sc->addr) \
222 fetch_memory_##type(regs, (void *)sc->addr, dest); \
223 else \
224 *(type *)dest = 0; \
225}
226DEFINE_BASIC_FETCH_FUNCS(symbol)
227DEFINE_FETCH_symbol(string)
228DEFINE_FETCH_symbol(string_size)
229
230
231#define fetch_file_offset_u8 NULL
232#define fetch_file_offset_u16 NULL
233#define fetch_file_offset_u32 NULL
234#define fetch_file_offset_u64 NULL
235#define fetch_file_offset_string NULL
236#define fetch_file_offset_string_size NULL
237
238
239const struct fetch_type kprobes_fetch_type_table[] = {
240
241 [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
242 sizeof(u32), 1, "__data_loc char[]"),
243 [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
244 string_size, sizeof(u32), 0, "u32"),
245
246 ASSIGN_FETCH_TYPE(u8, u8, 0),
247 ASSIGN_FETCH_TYPE(u16, u16, 0),
248 ASSIGN_FETCH_TYPE(u32, u32, 0),
249 ASSIGN_FETCH_TYPE(u64, u64, 0),
250 ASSIGN_FETCH_TYPE(s8, u8, 1),
251 ASSIGN_FETCH_TYPE(s16, u16, 1),
252 ASSIGN_FETCH_TYPE(s32, u32, 1),
253 ASSIGN_FETCH_TYPE(s64, u64, 1),
254
255 ASSIGN_FETCH_TYPE_END
256};
257
258
259
260
261static struct trace_kprobe *alloc_trace_kprobe(const char *group,
262 const char *event,
263 void *addr,
264 const char *symbol,
265 unsigned long offs,
266 int nargs, bool is_return)
267{
268 struct trace_kprobe *tk;
269 int ret = -ENOMEM;
270
271 tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);
272 if (!tk)
273 return ERR_PTR(ret);
274
275 if (symbol) {
276 tk->symbol = kstrdup(symbol, GFP_KERNEL);
277 if (!tk->symbol)
278 goto error;
279 tk->rp.kp.symbol_name = tk->symbol;
280 tk->rp.kp.offset = offs;
281 } else
282 tk->rp.kp.addr = addr;
283
284 if (is_return)
285 tk->rp.handler = kretprobe_dispatcher;
286 else
287 tk->rp.kp.pre_handler = kprobe_dispatcher;
288
289 if (!event || !is_good_name(event)) {
290 ret = -EINVAL;
291 goto error;
292 }
293
294 tk->tp.call.class = &tk->tp.class;
295 tk->tp.call.name = kstrdup(event, GFP_KERNEL);
296 if (!tk->tp.call.name)
297 goto error;
298
299 if (!group || !is_good_name(group)) {
300 ret = -EINVAL;
301 goto error;
302 }
303
304 tk->tp.class.system = kstrdup(group, GFP_KERNEL);
305 if (!tk->tp.class.system)
306 goto error;
307
308 INIT_LIST_HEAD(&tk->list);
309 INIT_LIST_HEAD(&tk->tp.files);
310 return tk;
311error:
312 kfree(tk->tp.call.name);
313 kfree(tk->symbol);
314 kfree(tk);
315 return ERR_PTR(ret);
316}
317
318static void free_trace_kprobe(struct trace_kprobe *tk)
319{
320 int i;
321
322 for (i = 0; i < tk->tp.nr_args; i++)
323 traceprobe_free_probe_arg(&tk->tp.args[i]);
324
325 kfree(tk->tp.call.class->system);
326 kfree(tk->tp.call.name);
327 kfree(tk->symbol);
328 kfree(tk);
329}
330
331static struct trace_kprobe *find_trace_kprobe(const char *event,
332 const char *group)
333{
334 struct trace_kprobe *tk;
335
336 list_for_each_entry(tk, &probe_list, list)
337 if (strcmp(tk->tp.call.name, event) == 0 &&
338 strcmp(tk->tp.call.class->system, group) == 0)
339 return tk;
340 return NULL;
341}
342
343
344
345
346
347static int
348enable_trace_kprobe(struct trace_kprobe *tk, struct ftrace_event_file *file)
349{
350 int ret = 0;
351
352 if (file) {
353 struct event_file_link *link;
354
355 link = kmalloc(sizeof(*link), GFP_KERNEL);
356 if (!link) {
357 ret = -ENOMEM;
358 goto out;
359 }
360
361 link->file = file;
362 list_add_tail_rcu(&link->list, &tk->tp.files);
363
364 tk->tp.flags |= TP_FLAG_TRACE;
365 } else
366 tk->tp.flags |= TP_FLAG_PROFILE;
367
368 if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {
369 if (trace_kprobe_is_return(tk))
370 ret = enable_kretprobe(&tk->rp);
371 else
372 ret = enable_kprobe(&tk->rp.kp);
373 }
374 out:
375 return ret;
376}
377
378static struct event_file_link *
379find_event_file_link(struct trace_probe *tp, struct ftrace_event_file *file)
380{
381 struct event_file_link *link;
382
383 list_for_each_entry(link, &tp->files, list)
384 if (link->file == file)
385 return link;
386
387 return NULL;
388}
389
390
391
392
393
394static int
395disable_trace_kprobe(struct trace_kprobe *tk, struct ftrace_event_file *file)
396{
397 struct event_file_link *link = NULL;
398 int wait = 0;
399 int ret = 0;
400
401 if (file) {
402 link = find_event_file_link(&tk->tp, file);
403 if (!link) {
404 ret = -EINVAL;
405 goto out;
406 }
407
408 list_del_rcu(&link->list);
409 wait = 1;
410 if (!list_empty(&tk->tp.files))
411 goto out;
412
413 tk->tp.flags &= ~TP_FLAG_TRACE;
414 } else
415 tk->tp.flags &= ~TP_FLAG_PROFILE;
416
417 if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {
418 if (trace_kprobe_is_return(tk))
419 disable_kretprobe(&tk->rp);
420 else
421 disable_kprobe(&tk->rp.kp);
422 wait = 1;
423 }
424
425
426
427
428
429
430 if (list_empty(&tk->list))
431 wait = 0;
432 out:
433 if (wait) {
434
435
436
437
438
439
440
441
442 synchronize_sched();
443 kfree(link);
444 }
445
446 return ret;
447}
448
449
450static int __register_trace_kprobe(struct trace_kprobe *tk)
451{
452 int i, ret;
453
454 if (trace_probe_is_registered(&tk->tp))
455 return -EINVAL;
456
457 for (i = 0; i < tk->tp.nr_args; i++)
458 traceprobe_update_arg(&tk->tp.args[i]);
459
460
461 if (trace_probe_is_enabled(&tk->tp))
462 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
463 else
464 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
465
466 if (trace_kprobe_is_return(tk))
467 ret = register_kretprobe(&tk->rp);
468 else
469 ret = register_kprobe(&tk->rp.kp);
470
471 if (ret == 0)
472 tk->tp.flags |= TP_FLAG_REGISTERED;
473 else {
474 pr_warning("Could not insert probe at %s+%lu: %d\n",
475 trace_kprobe_symbol(tk), trace_kprobe_offset(tk), ret);
476 if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) {
477 pr_warning("This probe might be able to register after"
478 "target module is loaded. Continue.\n");
479 ret = 0;
480 } else if (ret == -EILSEQ) {
481 pr_warning("Probing address(0x%p) is not an "
482 "instruction boundary.\n",
483 tk->rp.kp.addr);
484 ret = -EINVAL;
485 }
486 }
487
488 return ret;
489}
490
491
492static void __unregister_trace_kprobe(struct trace_kprobe *tk)
493{
494 if (trace_probe_is_registered(&tk->tp)) {
495 if (trace_kprobe_is_return(tk))
496 unregister_kretprobe(&tk->rp);
497 else
498 unregister_kprobe(&tk->rp.kp);
499 tk->tp.flags &= ~TP_FLAG_REGISTERED;
500
501 if (tk->rp.kp.symbol_name)
502 tk->rp.kp.addr = NULL;
503 }
504}
505
506
507static int unregister_trace_kprobe(struct trace_kprobe *tk)
508{
509
510 if (trace_probe_is_enabled(&tk->tp))
511 return -EBUSY;
512
513
514 if (unregister_kprobe_event(tk))
515 return -EBUSY;
516
517 __unregister_trace_kprobe(tk);
518 list_del(&tk->list);
519
520 return 0;
521}
522
523
524static int register_trace_kprobe(struct trace_kprobe *tk)
525{
526 struct trace_kprobe *old_tk;
527 int ret;
528
529 mutex_lock(&probe_lock);
530
531
532 old_tk = find_trace_kprobe(tk->tp.call.name, tk->tp.call.class->system);
533 if (old_tk) {
534 ret = unregister_trace_kprobe(old_tk);
535 if (ret < 0)
536 goto end;
537 free_trace_kprobe(old_tk);
538 }
539
540
541 ret = register_kprobe_event(tk);
542 if (ret) {
543 pr_warning("Failed to register probe event(%d)\n", ret);
544 goto end;
545 }
546
547
548 ret = __register_trace_kprobe(tk);
549 if (ret < 0)
550 unregister_kprobe_event(tk);
551 else
552 list_add_tail(&tk->list, &probe_list);
553
554end:
555 mutex_unlock(&probe_lock);
556 return ret;
557}
558
559
560static int trace_kprobe_module_callback(struct notifier_block *nb,
561 unsigned long val, void *data)
562{
563 struct module *mod = data;
564 struct trace_kprobe *tk;
565 int ret;
566
567 if (val != MODULE_STATE_COMING)
568 return NOTIFY_DONE;
569
570
571 mutex_lock(&probe_lock);
572 list_for_each_entry(tk, &probe_list, list) {
573 if (trace_kprobe_within_module(tk, mod)) {
574
575 __unregister_trace_kprobe(tk);
576 ret = __register_trace_kprobe(tk);
577 if (ret)
578 pr_warning("Failed to re-register probe %s on"
579 "%s: %d\n",
580 tk->tp.call.name, mod->name, ret);
581 }
582 }
583 mutex_unlock(&probe_lock);
584
585 return NOTIFY_DONE;
586}
587
588static struct notifier_block trace_kprobe_module_nb = {
589 .notifier_call = trace_kprobe_module_callback,
590 .priority = 1
591};
592
593static int create_trace_kprobe(int argc, char **argv)
594{
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613 struct trace_kprobe *tk;
614 int i, ret = 0;
615 bool is_return = false, is_delete = false;
616 char *symbol = NULL, *event = NULL, *group = NULL;
617 char *arg;
618 unsigned long offset = 0;
619 void *addr = NULL;
620 char buf[MAX_EVENT_NAME_LEN];
621
622
623 if (argv[0][0] == 'p')
624 is_return = false;
625 else if (argv[0][0] == 'r')
626 is_return = true;
627 else if (argv[0][0] == '-')
628 is_delete = true;
629 else {
630 pr_info("Probe definition must be started with 'p', 'r' or"
631 " '-'.\n");
632 return -EINVAL;
633 }
634
635 if (argv[0][1] == ':') {
636 event = &argv[0][2];
637 if (strchr(event, '/')) {
638 group = event;
639 event = strchr(group, '/') + 1;
640 event[-1] = '\0';
641 if (strlen(group) == 0) {
642 pr_info("Group name is not specified\n");
643 return -EINVAL;
644 }
645 }
646 if (strlen(event) == 0) {
647 pr_info("Event name is not specified\n");
648 return -EINVAL;
649 }
650 }
651 if (!group)
652 group = KPROBE_EVENT_SYSTEM;
653
654 if (is_delete) {
655 if (!event) {
656 pr_info("Delete command needs an event name.\n");
657 return -EINVAL;
658 }
659 mutex_lock(&probe_lock);
660 tk = find_trace_kprobe(event, group);
661 if (!tk) {
662 mutex_unlock(&probe_lock);
663 pr_info("Event %s/%s doesn't exist.\n", group, event);
664 return -ENOENT;
665 }
666
667 ret = unregister_trace_kprobe(tk);
668 if (ret == 0)
669 free_trace_kprobe(tk);
670 mutex_unlock(&probe_lock);
671 return ret;
672 }
673
674 if (argc < 2) {
675 pr_info("Probe point is not specified.\n");
676 return -EINVAL;
677 }
678
679
680
681 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
682
683 symbol = argv[1];
684
685 ret = traceprobe_split_symbol_offset(symbol, &offset);
686 if (ret) {
687 pr_info("Failed to parse either an address or a symbol.\n");
688 return ret;
689 }
690 }
691 argc -= 2; argv += 2;
692
693
694 if (!event) {
695
696 if (symbol)
697 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
698 is_return ? 'r' : 'p', symbol, offset);
699 else
700 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
701 is_return ? 'r' : 'p', addr);
702 event = buf;
703 }
704 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, argc,
705 is_return);
706 if (IS_ERR(tk)) {
707 pr_info("Failed to allocate trace_probe.(%d)\n",
708 (int)PTR_ERR(tk));
709 return PTR_ERR(tk);
710 }
711
712
713 ret = 0;
714 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
715 struct probe_arg *parg = &tk->tp.args[i];
716
717
718 tk->tp.nr_args++;
719
720
721 arg = strchr(argv[i], '=');
722 if (arg) {
723 *arg++ = '\0';
724 parg->name = kstrdup(argv[i], GFP_KERNEL);
725 } else {
726 arg = argv[i];
727
728 snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
729 parg->name = kstrdup(buf, GFP_KERNEL);
730 }
731
732 if (!parg->name) {
733 pr_info("Failed to allocate argument[%d] name.\n", i);
734 ret = -ENOMEM;
735 goto error;
736 }
737
738 if (!is_good_name(parg->name)) {
739 pr_info("Invalid argument[%d] name: %s\n",
740 i, parg->name);
741 ret = -EINVAL;
742 goto error;
743 }
744
745 if (traceprobe_conflict_field_name(parg->name,
746 tk->tp.args, i)) {
747 pr_info("Argument[%d] name '%s' conflicts with "
748 "another field.\n", i, argv[i]);
749 ret = -EINVAL;
750 goto error;
751 }
752
753
754 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
755 is_return, true);
756 if (ret) {
757 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
758 goto error;
759 }
760 }
761
762 ret = register_trace_kprobe(tk);
763 if (ret)
764 goto error;
765 return 0;
766
767error:
768 free_trace_kprobe(tk);
769 return ret;
770}
771
772static int release_all_trace_kprobes(void)
773{
774 struct trace_kprobe *tk;
775 int ret = 0;
776
777 mutex_lock(&probe_lock);
778
779 list_for_each_entry(tk, &probe_list, list)
780 if (trace_probe_is_enabled(&tk->tp)) {
781 ret = -EBUSY;
782 goto end;
783 }
784
785 while (!list_empty(&probe_list)) {
786 tk = list_entry(probe_list.next, struct trace_kprobe, list);
787 ret = unregister_trace_kprobe(tk);
788 if (ret)
789 goto end;
790 free_trace_kprobe(tk);
791 }
792
793end:
794 mutex_unlock(&probe_lock);
795
796 return ret;
797}
798
799
800static void *probes_seq_start(struct seq_file *m, loff_t *pos)
801{
802 mutex_lock(&probe_lock);
803 return seq_list_start(&probe_list, *pos);
804}
805
806static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
807{
808 return seq_list_next(v, &probe_list, pos);
809}
810
811static void probes_seq_stop(struct seq_file *m, void *v)
812{
813 mutex_unlock(&probe_lock);
814}
815
816static int probes_seq_show(struct seq_file *m, void *v)
817{
818 struct trace_kprobe *tk = v;
819 int i;
820
821 seq_printf(m, "%c", trace_kprobe_is_return(tk) ? 'r' : 'p');
822 seq_printf(m, ":%s/%s", tk->tp.call.class->system, tk->tp.call.name);
823
824 if (!tk->symbol)
825 seq_printf(m, " 0x%p", tk->rp.kp.addr);
826 else if (tk->rp.kp.offset)
827 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
828 tk->rp.kp.offset);
829 else
830 seq_printf(m, " %s", trace_kprobe_symbol(tk));
831
832 for (i = 0; i < tk->tp.nr_args; i++)
833 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
834 seq_printf(m, "\n");
835
836 return 0;
837}
838
839static const struct seq_operations probes_seq_op = {
840 .start = probes_seq_start,
841 .next = probes_seq_next,
842 .stop = probes_seq_stop,
843 .show = probes_seq_show
844};
845
846static int probes_open(struct inode *inode, struct file *file)
847{
848 int ret;
849
850 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
851 ret = release_all_trace_kprobes();
852 if (ret < 0)
853 return ret;
854 }
855
856 return seq_open(file, &probes_seq_op);
857}
858
859static ssize_t probes_write(struct file *file, const char __user *buffer,
860 size_t count, loff_t *ppos)
861{
862 return traceprobe_probes_write(file, buffer, count, ppos,
863 create_trace_kprobe);
864}
865
866static const struct file_operations kprobe_events_ops = {
867 .owner = THIS_MODULE,
868 .open = probes_open,
869 .read = seq_read,
870 .llseek = seq_lseek,
871 .release = seq_release,
872 .write = probes_write,
873};
874
875
876static int probes_profile_seq_show(struct seq_file *m, void *v)
877{
878 struct trace_kprobe *tk = v;
879
880 seq_printf(m, " %-44s %15lu %15lu\n", tk->tp.call.name, tk->nhit,
881 tk->rp.kp.nmissed);
882
883 return 0;
884}
885
886static const struct seq_operations profile_seq_op = {
887 .start = probes_seq_start,
888 .next = probes_seq_next,
889 .stop = probes_seq_stop,
890 .show = probes_profile_seq_show
891};
892
893static int profile_open(struct inode *inode, struct file *file)
894{
895 return seq_open(file, &profile_seq_op);
896}
897
898static const struct file_operations kprobe_profile_ops = {
899 .owner = THIS_MODULE,
900 .open = profile_open,
901 .read = seq_read,
902 .llseek = seq_lseek,
903 .release = seq_release,
904};
905
906
907static __kprobes void
908__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
909 struct ftrace_event_file *ftrace_file)
910{
911 struct kprobe_trace_entry_head *entry;
912 struct ring_buffer_event *event;
913 struct ring_buffer *buffer;
914 int size, dsize, pc;
915 unsigned long irq_flags;
916 struct ftrace_event_call *call = &tk->tp.call;
917
918 WARN_ON(call != ftrace_file->event_call);
919
920 if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
921 return;
922
923 local_save_flags(irq_flags);
924 pc = preempt_count();
925
926 dsize = __get_data_size(&tk->tp, regs);
927 size = sizeof(*entry) + tk->tp.size + dsize;
928
929 event = trace_event_buffer_lock_reserve(&buffer, ftrace_file,
930 call->event.type,
931 size, irq_flags, pc);
932 if (!event)
933 return;
934
935 entry = ring_buffer_event_data(event);
936 entry->ip = (unsigned long)tk->rp.kp.addr;
937 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
938
939 if (!filter_current_check_discard(buffer, call, entry, event))
940 trace_buffer_unlock_commit_regs(buffer, event,
941 irq_flags, pc, regs);
942}
943
944static __kprobes void
945kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
946{
947 struct event_file_link *link;
948
949 list_for_each_entry_rcu(link, &tk->tp.files, list)
950 __kprobe_trace_func(tk, regs, link->file);
951}
952
953
954static __kprobes void
955__kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
956 struct pt_regs *regs,
957 struct ftrace_event_file *ftrace_file)
958{
959 struct kretprobe_trace_entry_head *entry;
960 struct ring_buffer_event *event;
961 struct ring_buffer *buffer;
962 int size, pc, dsize;
963 unsigned long irq_flags;
964 struct ftrace_event_call *call = &tk->tp.call;
965
966 WARN_ON(call != ftrace_file->event_call);
967
968 if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
969 return;
970
971 local_save_flags(irq_flags);
972 pc = preempt_count();
973
974 dsize = __get_data_size(&tk->tp, regs);
975 size = sizeof(*entry) + tk->tp.size + dsize;
976
977 event = trace_event_buffer_lock_reserve(&buffer, ftrace_file,
978 call->event.type,
979 size, irq_flags, pc);
980 if (!event)
981 return;
982
983 entry = ring_buffer_event_data(event);
984 entry->func = (unsigned long)tk->rp.kp.addr;
985 entry->ret_ip = (unsigned long)ri->ret_addr;
986 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
987
988 if (!filter_current_check_discard(buffer, call, entry, event))
989 trace_buffer_unlock_commit_regs(buffer, event,
990 irq_flags, pc, regs);
991}
992
993static __kprobes void
994kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
995 struct pt_regs *regs)
996{
997 struct event_file_link *link;
998
999 list_for_each_entry_rcu(link, &tk->tp.files, list)
1000 __kretprobe_trace_func(tk, ri, regs, link->file);
1001}
1002
1003
1004static enum print_line_t
1005print_kprobe_event(struct trace_iterator *iter, int flags,
1006 struct trace_event *event)
1007{
1008 struct kprobe_trace_entry_head *field;
1009 struct trace_seq *s = &iter->seq;
1010 struct trace_probe *tp;
1011 u8 *data;
1012 int i;
1013
1014 field = (struct kprobe_trace_entry_head *)iter->ent;
1015 tp = container_of(event, struct trace_probe, call.event);
1016
1017 if (!trace_seq_printf(s, "%s: (", tp->call.name))
1018 goto partial;
1019
1020 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
1021 goto partial;
1022
1023 if (!trace_seq_puts(s, ")"))
1024 goto partial;
1025
1026 data = (u8 *)&field[1];
1027 for (i = 0; i < tp->nr_args; i++)
1028 if (!tp->args[i].type->print(s, tp->args[i].name,
1029 data + tp->args[i].offset, field))
1030 goto partial;
1031
1032 if (!trace_seq_puts(s, "\n"))
1033 goto partial;
1034
1035 return TRACE_TYPE_HANDLED;
1036partial:
1037 return TRACE_TYPE_PARTIAL_LINE;
1038}
1039
1040static enum print_line_t
1041print_kretprobe_event(struct trace_iterator *iter, int flags,
1042 struct trace_event *event)
1043{
1044 struct kretprobe_trace_entry_head *field;
1045 struct trace_seq *s = &iter->seq;
1046 struct trace_probe *tp;
1047 u8 *data;
1048 int i;
1049
1050 field = (struct kretprobe_trace_entry_head *)iter->ent;
1051 tp = container_of(event, struct trace_probe, call.event);
1052
1053 if (!trace_seq_printf(s, "%s: (", tp->call.name))
1054 goto partial;
1055
1056 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
1057 goto partial;
1058
1059 if (!trace_seq_puts(s, " <- "))
1060 goto partial;
1061
1062 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
1063 goto partial;
1064
1065 if (!trace_seq_puts(s, ")"))
1066 goto partial;
1067
1068 data = (u8 *)&field[1];
1069 for (i = 0; i < tp->nr_args; i++)
1070 if (!tp->args[i].type->print(s, tp->args[i].name,
1071 data + tp->args[i].offset, field))
1072 goto partial;
1073
1074 if (!trace_seq_puts(s, "\n"))
1075 goto partial;
1076
1077 return TRACE_TYPE_HANDLED;
1078partial:
1079 return TRACE_TYPE_PARTIAL_LINE;
1080}
1081
1082
1083static struct trace_kprobe* trace_kprobe(struct ftrace_event_call *call)
1084{
1085 return call->rh_data->data;
1086}
1087
1088static int kprobe_event_define_fields(struct ftrace_event_call *event_call)
1089{
1090 int ret, i;
1091 struct kprobe_trace_entry_head field;
1092 struct trace_kprobe *tk = trace_kprobe(event_call);
1093
1094 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1095
1096 for (i = 0; i < tk->tp.nr_args; i++) {
1097 struct probe_arg *parg = &tk->tp.args[i];
1098
1099 ret = trace_define_field(event_call, parg->type->fmttype,
1100 parg->name,
1101 sizeof(field) + parg->offset,
1102 parg->type->size,
1103 parg->type->is_signed,
1104 FILTER_OTHER);
1105 if (ret)
1106 return ret;
1107 }
1108 return 0;
1109}
1110
1111static int kretprobe_event_define_fields(struct ftrace_event_call *event_call)
1112{
1113 int ret, i;
1114 struct kretprobe_trace_entry_head field;
1115 struct trace_kprobe *tk = trace_kprobe(event_call);
1116
1117 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1118 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1119
1120 for (i = 0; i < tk->tp.nr_args; i++) {
1121 struct probe_arg *parg = &tk->tp.args[i];
1122
1123 ret = trace_define_field(event_call, parg->type->fmttype,
1124 parg->name,
1125 sizeof(field) + parg->offset,
1126 parg->type->size,
1127 parg->type->is_signed,
1128 FILTER_OTHER);
1129 if (ret)
1130 return ret;
1131 }
1132 return 0;
1133}
1134
1135#ifdef CONFIG_PERF_EVENTS
1136
1137
1138static __kprobes void
1139kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1140{
1141 struct ftrace_event_call *call = &tk->tp.call;
1142 struct kprobe_trace_entry_head *entry;
1143 struct hlist_head *head;
1144 int size, __size, dsize;
1145 int rctx;
1146
1147 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
1148 return;
1149
1150 dsize = __get_data_size(&tk->tp, regs);
1151 __size = sizeof(*entry) + tk->tp.size + dsize;
1152 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1153 size -= sizeof(u32);
1154 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
1155 "profile buffer not large enough"))
1156 return;
1157
1158 entry = perf_trace_buf_prepare(size, call->event.type, NULL, &rctx);
1159 if (!entry)
1160 return;
1161
1162 entry->ip = (unsigned long)tk->rp.kp.addr;
1163 memset(&entry[1], 0, dsize);
1164 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1165
1166 head = this_cpu_ptr(call->perf_events);
1167 perf_trace_buf_submit(entry, size, rctx,
1168 entry->ip, 1, regs, head, NULL);
1169}
1170
1171
1172static __kprobes void
1173kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1174 struct pt_regs *regs)
1175{
1176 struct ftrace_event_call *call = &tk->tp.call;
1177 struct kretprobe_trace_entry_head *entry;
1178 struct hlist_head *head;
1179 int size, __size, dsize;
1180 int rctx;
1181
1182 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
1183 return;
1184
1185 dsize = __get_data_size(&tk->tp, regs);
1186 __size = sizeof(*entry) + tk->tp.size + dsize;
1187 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1188 size -= sizeof(u32);
1189 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
1190 "profile buffer not large enough"))
1191 return;
1192
1193 entry = perf_trace_buf_prepare(size, call->event.type, NULL, &rctx);
1194 if (!entry)
1195 return;
1196
1197 entry->func = (unsigned long)tk->rp.kp.addr;
1198 entry->ret_ip = (unsigned long)ri->ret_addr;
1199 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1200
1201 head = this_cpu_ptr(call->perf_events);
1202 perf_trace_buf_submit(entry, size, rctx,
1203 entry->ret_ip, 1, regs, head, NULL);
1204}
1205#endif
1206
1207
1208
1209
1210
1211
1212
1213static __kprobes
1214int kprobe_register(struct ftrace_event_call *event,
1215 enum trace_reg type, void *data)
1216{
1217 struct trace_kprobe *tk = trace_kprobe(event);
1218 struct ftrace_event_file *file = data;
1219
1220 switch (type) {
1221 case TRACE_REG_REGISTER:
1222 return enable_trace_kprobe(tk, file);
1223 case TRACE_REG_UNREGISTER:
1224 return disable_trace_kprobe(tk, file);
1225
1226#ifdef CONFIG_PERF_EVENTS
1227 case TRACE_REG_PERF_REGISTER:
1228 return enable_trace_kprobe(tk, NULL);
1229 case TRACE_REG_PERF_UNREGISTER:
1230 return disable_trace_kprobe(tk, NULL);
1231 case TRACE_REG_PERF_OPEN:
1232 case TRACE_REG_PERF_CLOSE:
1233 case TRACE_REG_PERF_ADD:
1234 case TRACE_REG_PERF_DEL:
1235 return 0;
1236#endif
1237 }
1238 return 0;
1239}
1240
1241static __kprobes
1242int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1243{
1244 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1245
1246 tk->nhit++;
1247
1248 if (tk->tp.flags & TP_FLAG_TRACE)
1249 kprobe_trace_func(tk, regs);
1250#ifdef CONFIG_PERF_EVENTS
1251 if (tk->tp.flags & TP_FLAG_PROFILE)
1252 kprobe_perf_func(tk, regs);
1253#endif
1254 return 0;
1255}
1256
1257static __kprobes
1258int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1259{
1260 struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
1261
1262 tk->nhit++;
1263
1264 if (tk->tp.flags & TP_FLAG_TRACE)
1265 kretprobe_trace_func(tk, ri, regs);
1266#ifdef CONFIG_PERF_EVENTS
1267 if (tk->tp.flags & TP_FLAG_PROFILE)
1268 kretprobe_perf_func(tk, ri, regs);
1269#endif
1270 return 0;
1271}
1272
1273static struct trace_event_functions kretprobe_funcs = {
1274 .trace = print_kretprobe_event
1275};
1276
1277static struct trace_event_functions kprobe_funcs = {
1278 .trace = print_kprobe_event
1279};
1280
1281static inline void init_trace_event_call(struct trace_kprobe *tk,
1282 struct ftrace_event_call *call)
1283{
1284 INIT_LIST_HEAD(&call->class->fields);
1285 if (trace_kprobe_is_return(tk)) {
1286 call->event.funcs = &kretprobe_funcs;
1287 call->class->define_fields = kretprobe_event_define_fields;
1288 } else {
1289 call->event.funcs = &kprobe_funcs;
1290 call->class->define_fields = kprobe_event_define_fields;
1291 }
1292
1293 call->flags = TRACE_EVENT_FL_KPROBE;
1294 call->class->reg = kprobe_register;
1295 call->rh_data = (void *) tk;
1296}
1297
1298static int register_kprobe_event(struct trace_kprobe *tk)
1299{
1300 struct ftrace_event_call *call = &tk->tp.call;
1301 int ret = 0;
1302
1303 init_trace_event_call(tk, call);
1304
1305 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)
1306 return -ENOMEM;
1307 ret = register_ftrace_event(&call->event);
1308 if (!ret) {
1309 kfree(call->print_fmt);
1310 return -ENODEV;
1311 }
1312 ret = trace_add_event_call(call);
1313 if (ret) {
1314 pr_info("Failed to register kprobe event: %s\n", call->name);
1315 kfree(call->print_fmt);
1316 unregister_ftrace_event(&call->event);
1317 }
1318 return ret;
1319}
1320
1321static int unregister_kprobe_event(struct trace_kprobe *tk)
1322{
1323 int ret;
1324
1325
1326 ret = trace_remove_event_call(&tk->tp.call);
1327 if (!ret)
1328 kfree(tk->tp.call.print_fmt);
1329 return ret;
1330}
1331
1332#ifdef CONFIG_PERF_EVENTS
1333
1334struct ftrace_event_call *
1335create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1336 bool is_return)
1337{
1338 struct trace_kprobe *tk;
1339 int ret;
1340 char *event;
1341
1342
1343
1344
1345
1346
1347 event = func ? func : "DUMMY_EVENT";
1348
1349 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func,
1350 offs, 0 ,
1351 is_return);
1352
1353 if (IS_ERR(tk)) {
1354 pr_info("Failed to allocate trace_probe.(%d)\n",
1355 (int)PTR_ERR(tk));
1356 return ERR_CAST(tk);
1357 }
1358
1359 init_trace_event_call(tk, &tk->tp.call);
1360
1361 if (alloc_rh_data(&tk->tp.call)) {
1362 ret = -ENOMEM;
1363 goto error;
1364 }
1365
1366 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) {
1367 ret = -ENOMEM;
1368 destroy_rh_data(&tk->tp.call);
1369 goto error;
1370 }
1371
1372 ret = __register_trace_kprobe(tk);
1373 if (ret < 0) {
1374 destroy_rh_data(&tk->tp.call);
1375 kfree(tk->tp.call.print_fmt);
1376 goto error;
1377 }
1378
1379 return &tk->tp.call;
1380error:
1381 free_trace_kprobe(tk);
1382 return ERR_PTR(ret);
1383}
1384
1385void destroy_local_trace_kprobe(struct ftrace_event_call *event_call)
1386{
1387 struct trace_kprobe *tk;
1388
1389 tk = container_of(event_call, struct trace_kprobe, tp.call);
1390
1391 if (trace_probe_is_enabled(&tk->tp)) {
1392 WARN_ON(1);
1393 return;
1394 }
1395
1396 __unregister_trace_kprobe(tk);
1397 destroy_rh_data(event_call);
1398
1399 kfree(tk->tp.call.print_fmt);
1400 free_trace_kprobe(tk);
1401}
1402#endif
1403
1404
1405static __init int init_kprobe_trace(void)
1406{
1407 struct dentry *d_tracer;
1408 struct dentry *entry;
1409
1410 if (register_module_notifier(&trace_kprobe_module_nb))
1411 return -EINVAL;
1412
1413 d_tracer = tracing_init_dentry();
1414 if (!d_tracer)
1415 return 0;
1416
1417 entry = debugfs_create_file("kprobe_events", 0644, d_tracer,
1418 NULL, &kprobe_events_ops);
1419
1420
1421 if (!entry)
1422 pr_warning("Could not create debugfs "
1423 "'kprobe_events' entry\n");
1424
1425
1426 entry = debugfs_create_file("kprobe_profile", 0444, d_tracer,
1427 NULL, &kprobe_profile_ops);
1428
1429 if (!entry)
1430 pr_warning("Could not create debugfs "
1431 "'kprobe_profile' entry\n");
1432 return 0;
1433}
1434fs_initcall(init_kprobe_trace);
1435
1436
1437#ifdef CONFIG_FTRACE_STARTUP_TEST
1438
1439
1440
1441
1442
1443static __used int kprobe_trace_selftest_target(int a1, int a2, int a3,
1444 int a4, int a5, int a6)
1445{
1446 return a1 + a2 + a3 + a4 + a5 + a6;
1447}
1448
1449static struct ftrace_event_file *
1450find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
1451{
1452 struct ftrace_event_file *file;
1453
1454 list_for_each_entry(file, &tr->events, list)
1455 if (file->event_call == &tk->tp.call)
1456 return file;
1457
1458 return NULL;
1459}
1460
1461
1462
1463
1464
1465static __init int kprobe_trace_self_tests_init(void)
1466{
1467 int ret, warn = 0;
1468 int (*target)(int, int, int, int, int, int);
1469 struct trace_kprobe *tk;
1470 struct ftrace_event_file *file;
1471
1472 target = kprobe_trace_selftest_target;
1473
1474 pr_info("Testing kprobe tracing: ");
1475
1476 ret = traceprobe_command("p:testprobe kprobe_trace_selftest_target "
1477 "$stack $stack0 +0($stack)",
1478 create_trace_kprobe);
1479 if (WARN_ON_ONCE(ret)) {
1480 pr_warn("error on probing function entry.\n");
1481 warn++;
1482 } else {
1483
1484 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1485 if (WARN_ON_ONCE(tk == NULL)) {
1486 pr_warn("error on getting new probe.\n");
1487 warn++;
1488 } else {
1489 file = find_trace_probe_file(tk, top_trace_array());
1490 if (WARN_ON_ONCE(file == NULL)) {
1491 pr_warn("error on getting probe file.\n");
1492 warn++;
1493 } else
1494 enable_trace_kprobe(tk, file);
1495 }
1496 }
1497
1498 ret = traceprobe_command("r:testprobe2 kprobe_trace_selftest_target "
1499 "$retval", create_trace_kprobe);
1500 if (WARN_ON_ONCE(ret)) {
1501 pr_warn("error on probing function return.\n");
1502 warn++;
1503 } else {
1504
1505 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1506 if (WARN_ON_ONCE(tk == NULL)) {
1507 pr_warn("error on getting 2nd new probe.\n");
1508 warn++;
1509 } else {
1510 file = find_trace_probe_file(tk, top_trace_array());
1511 if (WARN_ON_ONCE(file == NULL)) {
1512 pr_warn("error on getting probe file.\n");
1513 warn++;
1514 } else
1515 enable_trace_kprobe(tk, file);
1516 }
1517 }
1518
1519 if (warn)
1520 goto end;
1521
1522 ret = target(1, 2, 3, 4, 5, 6);
1523
1524
1525 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1526 if (WARN_ON_ONCE(tk == NULL)) {
1527 pr_warn("error on getting test probe.\n");
1528 warn++;
1529 } else {
1530 file = find_trace_probe_file(tk, top_trace_array());
1531 if (WARN_ON_ONCE(file == NULL)) {
1532 pr_warn("error on getting probe file.\n");
1533 warn++;
1534 } else
1535 disable_trace_kprobe(tk, file);
1536 }
1537
1538 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1539 if (WARN_ON_ONCE(tk == NULL)) {
1540 pr_warn("error on getting 2nd test probe.\n");
1541 warn++;
1542 } else {
1543 file = find_trace_probe_file(tk, top_trace_array());
1544 if (WARN_ON_ONCE(file == NULL)) {
1545 pr_warn("error on getting probe file.\n");
1546 warn++;
1547 } else
1548 disable_trace_kprobe(tk, file);
1549 }
1550
1551 ret = traceprobe_command("-:testprobe", create_trace_kprobe);
1552 if (WARN_ON_ONCE(ret)) {
1553 pr_warn("error on deleting a probe.\n");
1554 warn++;
1555 }
1556
1557 ret = traceprobe_command("-:testprobe2", create_trace_kprobe);
1558 if (WARN_ON_ONCE(ret)) {
1559 pr_warn("error on deleting a probe.\n");
1560 warn++;
1561 }
1562
1563end:
1564 release_all_trace_kprobes();
1565 if (warn)
1566 pr_cont("NG: Some tests are failed. Please check them.\n");
1567 else
1568 pr_cont("OK\n");
1569 return 0;
1570}
1571
1572late_initcall(kprobe_trace_self_tests_init);
1573
1574#endif
1575
