1#include <trace/syscall.h>
2#include <trace/events/syscalls.h>
3#include <linux/syscalls.h>
4#include <linux/slab.h>
5#include <linux/kernel.h>
6#include <linux/module.h>
7#include <linux/ftrace.h>
8#include <linux/perf_event.h>
9#include <asm/syscall.h>
10
11#include "trace_output.h"
12#include "trace.h"
13
14static DEFINE_MUTEX(syscall_trace_lock);
15
16static int syscall_enter_register(struct ftrace_event_call *event,
17 enum trace_reg type, void *data);
18static int syscall_exit_register(struct ftrace_event_call *event,
19 enum trace_reg type, void *data);
20
21static struct list_head *
22syscall_get_enter_fields(struct ftrace_event_call *call)
23{
24 struct syscall_metadata *entry = call->data;
25
26 return &entry->enter_fields;
27}
28
29extern struct syscall_metadata *__start_syscalls_metadata[];
30extern struct syscall_metadata *__stop_syscalls_metadata[];
31
32static struct syscall_metadata **syscalls_metadata;
33
34#ifndef ARCH_HAS_SYSCALL_MATCH_SYM_NAME
35static inline bool arch_syscall_match_sym_name(const char *sym, const char *name)
36{
37
38
39
40
41
42
43 return !strcmp(sym + 3, name + 3);
44}
45#endif
46
47#ifdef ARCH_TRACE_IGNORE_COMPAT_SYSCALLS
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63static int
64trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs)
65{
66 if (unlikely(arch_trace_is_compat_syscall(regs)))
67 return -1;
68
69 return syscall_get_nr(task, regs);
70}
71#else
72static inline int
73trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs)
74{
75 return syscall_get_nr(task, regs);
76}
77#endif
78
79static __init struct syscall_metadata *
80find_syscall_meta(unsigned long syscall)
81{
82 struct syscall_metadata **start;
83 struct syscall_metadata **stop;
84 char str[KSYM_SYMBOL_LEN];
85
86
87 start = __start_syscalls_metadata;
88 stop = __stop_syscalls_metadata;
89 kallsyms_lookup(syscall, NULL, NULL, NULL, str);
90
91 if (arch_syscall_match_sym_name(str, "sys_ni_syscall"))
92 return NULL;
93
94 for ( ; start < stop; start++) {
95 if ((*start)->name && arch_syscall_match_sym_name(str, (*start)->name))
96 return *start;
97 }
98 return NULL;
99}
100
101static struct syscall_metadata *syscall_nr_to_meta(int nr)
102{
103 if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
104 return NULL;
105
106 return syscalls_metadata[nr];
107}
108
109static enum print_line_t
110print_syscall_enter(struct trace_iterator *iter, int flags,
111 struct trace_event *event)
112{
113 struct trace_seq *s = &iter->seq;
114 struct trace_entry *ent = iter->ent;
115 struct syscall_trace_enter *trace;
116 struct syscall_metadata *entry;
117 int i, ret, syscall;
118
119 trace = (typeof(trace))ent;
120 syscall = trace->nr;
121 entry = syscall_nr_to_meta(syscall);
122
123 if (!entry)
124 goto end;
125
126 if (entry->enter_event->event.type != ent->type) {
127 WARN_ON_ONCE(1);
128 goto end;
129 }
130
131 ret = trace_seq_printf(s, "%s(", entry->name);
132 if (!ret)
133 return TRACE_TYPE_PARTIAL_LINE;
134
135 for (i = 0; i < entry->nb_args; i++) {
136
137 if (trace_flags & TRACE_ITER_VERBOSE) {
138 ret = trace_seq_printf(s, "%s ", entry->types[i]);
139 if (!ret)
140 return TRACE_TYPE_PARTIAL_LINE;
141 }
142
143 ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
144 trace->args[i],
145 i == entry->nb_args - 1 ? "" : ", ");
146 if (!ret)
147 return TRACE_TYPE_PARTIAL_LINE;
148 }
149
150 ret = trace_seq_putc(s, ')');
151 if (!ret)
152 return TRACE_TYPE_PARTIAL_LINE;
153
154end:
155 ret = trace_seq_putc(s, '\n');
156 if (!ret)
157 return TRACE_TYPE_PARTIAL_LINE;
158
159 return TRACE_TYPE_HANDLED;
160}
161
162static enum print_line_t
163print_syscall_exit(struct trace_iterator *iter, int flags,
164 struct trace_event *event)
165{
166 struct trace_seq *s = &iter->seq;
167 struct trace_entry *ent = iter->ent;
168 struct syscall_trace_exit *trace;
169 int syscall;
170 struct syscall_metadata *entry;
171 int ret;
172
173 trace = (typeof(trace))ent;
174 syscall = trace->nr;
175 entry = syscall_nr_to_meta(syscall);
176
177 if (!entry) {
178 trace_seq_putc(s, '\n');
179 return TRACE_TYPE_HANDLED;
180 }
181
182 if (entry->exit_event->event.type != ent->type) {
183 WARN_ON_ONCE(1);
184 return TRACE_TYPE_UNHANDLED;
185 }
186
187 ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
188 trace->ret);
189 if (!ret)
190 return TRACE_TYPE_PARTIAL_LINE;
191
192 return TRACE_TYPE_HANDLED;
193}
194
195extern char *__bad_type_size(void);
196
197#define SYSCALL_FIELD(type, name) \
198 sizeof(type) != sizeof(trace.name) ? \
199 __bad_type_size() : \
200 #type, #name, offsetof(typeof(trace), name), \
201 sizeof(trace.name), is_signed_type(type)
202
203static
204int __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len)
205{
206 int i;
207 int pos = 0;
208
209
210#define LEN_OR_ZERO (len ? len - pos : 0)
211
212 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
213 for (i = 0; i < entry->nb_args; i++) {
214 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
215 entry->args[i], sizeof(unsigned long),
216 i == entry->nb_args - 1 ? "" : ", ");
217 }
218 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
219
220 for (i = 0; i < entry->nb_args; i++) {
221 pos += snprintf(buf + pos, LEN_OR_ZERO,
222 ", ((unsigned long)(REC->%s))", entry->args[i]);
223 }
224
225#undef LEN_OR_ZERO
226
227
228 return pos;
229}
230
231static int set_syscall_print_fmt(struct ftrace_event_call *call)
232{
233 char *print_fmt;
234 int len;
235 struct syscall_metadata *entry = call->data;
236
237 if (entry->enter_event != call) {
238 call->print_fmt = "\"0x%lx\", REC->ret";
239 return 0;
240 }
241
242
243 len = __set_enter_print_fmt(entry, NULL, 0);
244
245 print_fmt = kmalloc(len + 1, GFP_KERNEL);
246 if (!print_fmt)
247 return -ENOMEM;
248
249
250 __set_enter_print_fmt(entry, print_fmt, len + 1);
251 call->print_fmt = print_fmt;
252
253 return 0;
254}
255
256static void free_syscall_print_fmt(struct ftrace_event_call *call)
257{
258 struct syscall_metadata *entry = call->data;
259
260 if (entry->enter_event == call)
261 kfree(call->print_fmt);
262}
263
264static int __init syscall_enter_define_fields(struct ftrace_event_call *call)
265{
266 struct syscall_trace_enter trace;
267 struct syscall_metadata *meta = call->data;
268 int ret;
269 int i;
270 int offset = offsetof(typeof(trace), args);
271
272 ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
273 if (ret)
274 return ret;
275
276 for (i = 0; i < meta->nb_args; i++) {
277 ret = trace_define_field(call, meta->types[i],
278 meta->args[i], offset,
279 sizeof(unsigned long), 0,
280 FILTER_OTHER);
281 offset += sizeof(unsigned long);
282 }
283
284 return ret;
285}
286
287static int __init syscall_exit_define_fields(struct ftrace_event_call *call)
288{
289 struct syscall_trace_exit trace;
290 int ret;
291
292 ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
293 if (ret)
294 return ret;
295
296 ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
297 FILTER_OTHER);
298
299 return ret;
300}
301
302static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)
303{
304 struct trace_array *tr = data;
305 struct syscall_trace_enter *entry;
306 struct syscall_metadata *sys_data;
307 struct ring_buffer_event *event;
308 struct ring_buffer *buffer;
309 unsigned long irq_flags;
310 int pc;
311 int syscall_nr;
312 int size;
313
314 syscall_nr = trace_get_syscall_nr(current, regs);
315 if (syscall_nr < 0)
316 return;
317 if (!test_bit(syscall_nr, tr->enabled_enter_syscalls))
318 return;
319
320 sys_data = syscall_nr_to_meta(syscall_nr);
321 if (!sys_data)
322 return;
323
324 size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;
325
326 local_save_flags(irq_flags);
327 pc = preempt_count();
328
329 buffer = tr->trace_buffer.buffer;
330 event = trace_buffer_lock_reserve(buffer,
331 sys_data->enter_event->event.type, size, irq_flags, pc);
332 if (!event)
333 return;
334
335 entry = ring_buffer_event_data(event);
336 entry->nr = syscall_nr;
337 syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
338
339 if (!filter_current_check_discard(buffer, sys_data->enter_event,
340 entry, event))
341 trace_current_buffer_unlock_commit(buffer, event,
342 irq_flags, pc);
343}
344
345static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)
346{
347 struct trace_array *tr = data;
348 struct syscall_trace_exit *entry;
349 struct syscall_metadata *sys_data;
350 struct ring_buffer_event *event;
351 struct ring_buffer *buffer;
352 unsigned long irq_flags;
353 int pc;
354 int syscall_nr;
355
356 syscall_nr = trace_get_syscall_nr(current, regs);
357 if (syscall_nr < 0)
358 return;
359 if (!test_bit(syscall_nr, tr->enabled_exit_syscalls))
360 return;
361
362 sys_data = syscall_nr_to_meta(syscall_nr);
363 if (!sys_data)
364 return;
365
366 local_save_flags(irq_flags);
367 pc = preempt_count();
368
369 buffer = tr->trace_buffer.buffer;
370 event = trace_buffer_lock_reserve(buffer,
371 sys_data->exit_event->event.type, sizeof(*entry),
372 irq_flags, pc);
373 if (!event)
374 return;
375
376 entry = ring_buffer_event_data(event);
377 entry->nr = syscall_nr;
378 entry->ret = syscall_get_return_value(current, regs);
379
380 if (!filter_current_check_discard(buffer, sys_data->exit_event,
381 entry, event))
382 trace_current_buffer_unlock_commit(buffer, event,
383 irq_flags, pc);
384}
385
386static int reg_event_syscall_enter(struct ftrace_event_file *file,
387 struct ftrace_event_call *call)
388{
389 struct trace_array *tr = file->tr;
390 int ret = 0;
391 int num;
392
393 num = ((struct syscall_metadata *)call->data)->syscall_nr;
394 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
395 return -ENOSYS;
396 mutex_lock(&syscall_trace_lock);
397 if (!tr->sys_refcount_enter)
398 ret = register_trace_sys_enter(ftrace_syscall_enter, tr);
399 if (!ret) {
400 set_bit(num, tr->enabled_enter_syscalls);
401 tr->sys_refcount_enter++;
402 }
403 mutex_unlock(&syscall_trace_lock);
404 return ret;
405}
406
407static void unreg_event_syscall_enter(struct ftrace_event_file *file,
408 struct ftrace_event_call *call)
409{
410 struct trace_array *tr = file->tr;
411 int num;
412
413 num = ((struct syscall_metadata *)call->data)->syscall_nr;
414 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
415 return;
416 mutex_lock(&syscall_trace_lock);
417 tr->sys_refcount_enter--;
418 clear_bit(num, tr->enabled_enter_syscalls);
419 if (!tr->sys_refcount_enter)
420 unregister_trace_sys_enter(ftrace_syscall_enter, tr);
421 mutex_unlock(&syscall_trace_lock);
422}
423
424static int reg_event_syscall_exit(struct ftrace_event_file *file,
425 struct ftrace_event_call *call)
426{
427 struct trace_array *tr = file->tr;
428 int ret = 0;
429 int num;
430
431 num = ((struct syscall_metadata *)call->data)->syscall_nr;
432 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
433 return -ENOSYS;
434 mutex_lock(&syscall_trace_lock);
435 if (!tr->sys_refcount_exit)
436 ret = register_trace_sys_exit(ftrace_syscall_exit, tr);
437 if (!ret) {
438 set_bit(num, tr->enabled_exit_syscalls);
439 tr->sys_refcount_exit++;
440 }
441 mutex_unlock(&syscall_trace_lock);
442 return ret;
443}
444
445static void unreg_event_syscall_exit(struct ftrace_event_file *file,
446 struct ftrace_event_call *call)
447{
448 struct trace_array *tr = file->tr;
449 int num;
450
451 num = ((struct syscall_metadata *)call->data)->syscall_nr;
452 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
453 return;
454 mutex_lock(&syscall_trace_lock);
455 tr->sys_refcount_exit--;
456 clear_bit(num, tr->enabled_exit_syscalls);
457 if (!tr->sys_refcount_exit)
458 unregister_trace_sys_exit(ftrace_syscall_exit, tr);
459 mutex_unlock(&syscall_trace_lock);
460}
461
462static int init_syscall_trace(struct ftrace_event_call *call)
463{
464 int id;
465 int num;
466
467 num = ((struct syscall_metadata *)call->data)->syscall_nr;
468 if (num < 0 || num >= NR_syscalls) {
469 pr_debug("syscall %s metadata not mapped, disabling ftrace event\n",
470 ((struct syscall_metadata *)call->data)->name);
471 return -ENOSYS;
472 }
473
474 if (set_syscall_print_fmt(call) < 0)
475 return -ENOMEM;
476
477 id = trace_event_raw_init(call);
478
479 if (id < 0) {
480 free_syscall_print_fmt(call);
481 return id;
482 }
483
484 return id;
485}
486
487struct trace_event_functions enter_syscall_print_funcs = {
488 .trace = print_syscall_enter,
489};
490
491struct trace_event_functions exit_syscall_print_funcs = {
492 .trace = print_syscall_exit,
493};
494
495struct ftrace_event_class __refdata event_class_syscall_enter = {
496 .system = "syscalls",
497 .reg = syscall_enter_register,
498 .define_fields = syscall_enter_define_fields,
499 .get_fields = syscall_get_enter_fields,
500 .raw_init = init_syscall_trace,
501};
502
503struct ftrace_event_class __refdata event_class_syscall_exit = {
504 .system = "syscalls",
505 .reg = syscall_exit_register,
506 .define_fields = syscall_exit_define_fields,
507 .fields = LIST_HEAD_INIT(event_class_syscall_exit.fields),
508 .raw_init = init_syscall_trace,
509};
510
511unsigned long __init __weak arch_syscall_addr(int nr)
512{
513 return (unsigned long)sys_call_table[nr];
514}
515
516static int __init init_ftrace_syscalls(void)
517{
518 struct syscall_metadata *meta;
519 unsigned long addr;
520 int i;
521
522 syscalls_metadata = kcalloc(NR_syscalls, sizeof(*syscalls_metadata),
523 GFP_KERNEL);
524 if (!syscalls_metadata) {
525 WARN_ON(1);
526 return -ENOMEM;
527 }
528
529 for (i = 0; i < NR_syscalls; i++) {
530 addr = arch_syscall_addr(i);
531 meta = find_syscall_meta(addr);
532 if (!meta)
533 continue;
534
535 meta->syscall_nr = i;
536 syscalls_metadata[i] = meta;
537 }
538
539 return 0;
540}
541early_initcall(init_ftrace_syscalls);
542
543#ifdef CONFIG_PERF_EVENTS
544
545static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
546static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
547static int sys_perf_refcount_enter;
548static int sys_perf_refcount_exit;
549
550static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
551{
552 struct syscall_metadata *sys_data;
553 struct syscall_trace_enter *rec;
554 struct hlist_head *head;
555 int syscall_nr;
556 int rctx;
557 int size;
558
559 syscall_nr = trace_get_syscall_nr(current, regs);
560 if (syscall_nr < 0)
561 return;
562 if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
563 return;
564
565 sys_data = syscall_nr_to_meta(syscall_nr);
566 if (!sys_data)
567 return;
568
569 head = this_cpu_ptr(sys_data->enter_event->perf_events);
570 if (hlist_empty(head))
571 return;
572
573
574 size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
575 size = ALIGN(size + sizeof(u32), sizeof(u64));
576 size -= sizeof(u32);
577
578 rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
579 sys_data->enter_event->event.type, regs, &rctx);
580 if (!rec)
581 return;
582
583 rec->nr = syscall_nr;
584 syscall_get_arguments(current, regs, 0, sys_data->nb_args,
585 (unsigned long *)&rec->args);
586 perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
587}
588
589static int perf_sysenter_enable(struct ftrace_event_call *call)
590{
591 int ret = 0;
592 int num;
593
594 num = ((struct syscall_metadata *)call->data)->syscall_nr;
595
596 mutex_lock(&syscall_trace_lock);
597 if (!sys_perf_refcount_enter)
598 ret = register_trace_sys_enter(perf_syscall_enter, NULL);
599 if (ret) {
600 pr_info("event trace: Could not activate"
601 "syscall entry trace point");
602 } else {
603 set_bit(num, enabled_perf_enter_syscalls);
604 sys_perf_refcount_enter++;
605 }
606 mutex_unlock(&syscall_trace_lock);
607 return ret;
608}
609
610static void perf_sysenter_disable(struct ftrace_event_call *call)
611{
612 int num;
613
614 num = ((struct syscall_metadata *)call->data)->syscall_nr;
615
616 mutex_lock(&syscall_trace_lock);
617 sys_perf_refcount_enter--;
618 clear_bit(num, enabled_perf_enter_syscalls);
619 if (!sys_perf_refcount_enter)
620 unregister_trace_sys_enter(perf_syscall_enter, NULL);
621 mutex_unlock(&syscall_trace_lock);
622}
623
624static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
625{
626 struct syscall_metadata *sys_data;
627 struct syscall_trace_exit *rec;
628 struct hlist_head *head;
629 int syscall_nr;
630 int rctx;
631 int size;
632
633 syscall_nr = trace_get_syscall_nr(current, regs);
634 if (syscall_nr < 0)
635 return;
636 if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
637 return;
638
639 sys_data = syscall_nr_to_meta(syscall_nr);
640 if (!sys_data)
641 return;
642
643 head = this_cpu_ptr(sys_data->exit_event->perf_events);
644 if (hlist_empty(head))
645 return;
646
647
648 size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
649 size -= sizeof(u32);
650
651 rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
652 sys_data->exit_event->event.type, regs, &rctx);
653 if (!rec)
654 return;
655
656 rec->nr = syscall_nr;
657 rec->ret = syscall_get_return_value(current, regs);
658 perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
659}
660
661static int perf_sysexit_enable(struct ftrace_event_call *call)
662{
663 int ret = 0;
664 int num;
665
666 num = ((struct syscall_metadata *)call->data)->syscall_nr;
667
668 mutex_lock(&syscall_trace_lock);
669 if (!sys_perf_refcount_exit)
670 ret = register_trace_sys_exit(perf_syscall_exit, NULL);
671 if (ret) {
672 pr_info("event trace: Could not activate"
673 "syscall exit trace point");
674 } else {
675 set_bit(num, enabled_perf_exit_syscalls);
676 sys_perf_refcount_exit++;
677 }
678 mutex_unlock(&syscall_trace_lock);
679 return ret;
680}
681
682static void perf_sysexit_disable(struct ftrace_event_call *call)
683{
684 int num;
685
686 num = ((struct syscall_metadata *)call->data)->syscall_nr;
687
688 mutex_lock(&syscall_trace_lock);
689 sys_perf_refcount_exit--;
690 clear_bit(num, enabled_perf_exit_syscalls);
691 if (!sys_perf_refcount_exit)
692 unregister_trace_sys_exit(perf_syscall_exit, NULL);
693 mutex_unlock(&syscall_trace_lock);
694}
695
696#endif
697
698static int syscall_enter_register(struct ftrace_event_call *event,
699 enum trace_reg type, void *data)
700{
701 struct ftrace_event_file *file = data;
702
703 switch (type) {
704 case TRACE_REG_REGISTER:
705 return reg_event_syscall_enter(file, event);
706 case TRACE_REG_UNREGISTER:
707 unreg_event_syscall_enter(file, event);
708 return 0;
709
710#ifdef CONFIG_PERF_EVENTS
711 case TRACE_REG_PERF_REGISTER:
712 return perf_sysenter_enable(event);
713 case TRACE_REG_PERF_UNREGISTER:
714 perf_sysenter_disable(event);
715 return 0;
716 case TRACE_REG_PERF_OPEN:
717 case TRACE_REG_PERF_CLOSE:
718 case TRACE_REG_PERF_ADD:
719 case TRACE_REG_PERF_DEL:
720 return 0;
721#endif
722 }
723 return 0;
724}
725
726static int syscall_exit_register(struct ftrace_event_call *event,
727 enum trace_reg type, void *data)
728{
729 struct ftrace_event_file *file = data;
730
731 switch (type) {
732 case TRACE_REG_REGISTER:
733 return reg_event_syscall_exit(file, event);
734 case TRACE_REG_UNREGISTER:
735 unreg_event_syscall_exit(file, event);
736 return 0;
737
738#ifdef CONFIG_PERF_EVENTS
739 case TRACE_REG_PERF_REGISTER:
740 return perf_sysexit_enable(event);
741 case TRACE_REG_PERF_UNREGISTER:
742 perf_sysexit_disable(event);
743 return 0;
744 case TRACE_REG_PERF_OPEN:
745 case TRACE_REG_PERF_CLOSE:
746 case TRACE_REG_PERF_ADD:
747 case TRACE_REG_PERF_DEL:
748 return 0;
749#endif
750 }
751 return 0;
752}
753