1
2
3
4
5
6
7
8
9
10
11
12#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13
14#include <linux/spinlock.h>
15#include <linux/hardirq.h>
16#include <linux/uaccess.h>
17#include <linux/ftrace.h>
18#include <linux/percpu.h>
19#include <linux/sched.h>
20#include <linux/slab.h>
21#include <linux/init.h>
22#include <linux/list.h>
23#include <linux/module.h>
24
25#include <trace/syscall.h>
26
27#include <asm/set_memory.h>
28#include <asm/kprobes.h>
29#include <asm/ftrace.h>
30#include <asm/nops.h>
31
32#ifdef CONFIG_DYNAMIC_FTRACE
33
34int ftrace_arch_code_modify_prepare(void)
35{
36 set_kernel_text_rw();
37 set_all_modules_text_rw();
38 return 0;
39}
40
41int ftrace_arch_code_modify_post_process(void)
42{
43 set_all_modules_text_ro();
44 set_kernel_text_ro();
45 return 0;
46}
47
48union ftrace_code_union {
49 char code[MCOUNT_INSN_SIZE];
50 struct {
51 unsigned char e8;
52 int offset;
53 } __attribute__((packed));
54};
55
56static int ftrace_calc_offset(long ip, long addr)
57{
58 return (int)(addr - ip);
59}
60
61static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
62{
63 static union ftrace_code_union calc;
64
65 calc.e8 = 0xe8;
66 calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
67
68
69
70
71
72 return calc.code;
73}
74
75static inline int
76within(unsigned long addr, unsigned long start, unsigned long end)
77{
78 return addr >= start && addr < end;
79}
80
81static unsigned long text_ip_addr(unsigned long ip)
82{
83
84
85
86
87
88
89
90
91 if (within(ip, (unsigned long)_text, (unsigned long)_etext))
92 ip = (unsigned long)__va(__pa_symbol(ip));
93
94 return ip;
95}
96
97static const unsigned char *ftrace_nop_replace(void)
98{
99 return ideal_nops[NOP_ATOMIC5];
100}
101
102static int
103ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
104 unsigned const char *new_code)
105{
106 unsigned char replaced[MCOUNT_INSN_SIZE];
107
108 ftrace_expected = old_code;
109
110
111
112
113
114
115
116
117
118
119 if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
120 return -EFAULT;
121
122
123 if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
124 return -EINVAL;
125
126 ip = text_ip_addr(ip);
127
128
129 if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
130 return -EPERM;
131
132 sync_core();
133
134 return 0;
135}
136
137int ftrace_make_nop(struct module *mod,
138 struct dyn_ftrace *rec, unsigned long addr)
139{
140 unsigned const char *new, *old;
141 unsigned long ip = rec->ip;
142
143 old = ftrace_call_replace(ip, addr);
144 new = ftrace_nop_replace();
145
146
147
148
149
150
151
152
153
154 if (addr == MCOUNT_ADDR)
155 return ftrace_modify_code_direct(rec->ip, old, new);
156
157 ftrace_expected = NULL;
158
159
160 WARN_ONCE(1, "invalid use of ftrace_make_nop");
161 return -EINVAL;
162}
163
164int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
165{
166 unsigned const char *new, *old;
167 unsigned long ip = rec->ip;
168
169 old = ftrace_nop_replace();
170 new = ftrace_call_replace(ip, addr);
171
172
173 return ftrace_modify_code_direct(rec->ip, old, new);
174}
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207atomic_t modifying_ftrace_code __read_mostly;
208
209static int
210ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
211 unsigned const char *new_code);
212
213
214
215
216
217
218
219
220
221int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
222 unsigned long addr)
223{
224 WARN_ON(1);
225 ftrace_expected = NULL;
226 return -EINVAL;
227}
228
229static unsigned long ftrace_update_func;
230
231static int update_ftrace_func(unsigned long ip, void *new)
232{
233 unsigned char old[MCOUNT_INSN_SIZE];
234 int ret;
235
236 memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
237
238 ftrace_update_func = ip;
239
240 smp_wmb();
241
242
243 atomic_inc(&modifying_ftrace_code);
244
245 ret = ftrace_modify_code(ip, old, new);
246
247 atomic_dec(&modifying_ftrace_code);
248
249 return ret;
250}
251
252int ftrace_update_ftrace_func(ftrace_func_t func)
253{
254 unsigned long ip = (unsigned long)(&ftrace_call);
255 unsigned char *new;
256 int ret;
257
258 new = ftrace_call_replace(ip, (unsigned long)func);
259 ret = update_ftrace_func(ip, new);
260
261
262 if (!ret) {
263 ip = (unsigned long)(&ftrace_regs_call);
264 new = ftrace_call_replace(ip, (unsigned long)func);
265 ret = update_ftrace_func(ip, new);
266 }
267
268 return ret;
269}
270
271static int is_ftrace_caller(unsigned long ip)
272{
273 if (ip == ftrace_update_func)
274 return 1;
275
276 return 0;
277}
278
279
280
281
282
283
284
285
286int ftrace_int3_handler(struct pt_regs *regs)
287{
288 unsigned long ip;
289
290 if (WARN_ON_ONCE(!regs))
291 return 0;
292
293 ip = regs->ip - 1;
294 if (!ftrace_location(ip) && !is_ftrace_caller(ip))
295 return 0;
296
297 regs->ip += MCOUNT_INSN_SIZE - 1;
298
299 return 1;
300}
301
302static int ftrace_write(unsigned long ip, const char *val, int size)
303{
304 ip = text_ip_addr(ip);
305
306 if (probe_kernel_write((void *)ip, val, size))
307 return -EPERM;
308
309 return 0;
310}
311
312static int add_break(unsigned long ip, const char *old)
313{
314 unsigned char replaced[MCOUNT_INSN_SIZE];
315 unsigned char brk = BREAKPOINT_INSTRUCTION;
316
317 if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
318 return -EFAULT;
319
320 ftrace_expected = old;
321
322
323 if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
324 return -EINVAL;
325
326 return ftrace_write(ip, &brk, 1);
327}
328
329static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
330{
331 unsigned const char *old;
332 unsigned long ip = rec->ip;
333
334 old = ftrace_call_replace(ip, addr);
335
336 return add_break(rec->ip, old);
337}
338
339
340static int add_brk_on_nop(struct dyn_ftrace *rec)
341{
342 unsigned const char *old;
343
344 old = ftrace_nop_replace();
345
346 return add_break(rec->ip, old);
347}
348
349static int add_breakpoints(struct dyn_ftrace *rec, int enable)
350{
351 unsigned long ftrace_addr;
352 int ret;
353
354 ftrace_addr = ftrace_get_addr_curr(rec);
355
356 ret = ftrace_test_record(rec, enable);
357
358 switch (ret) {
359 case FTRACE_UPDATE_IGNORE:
360 return 0;
361
362 case FTRACE_UPDATE_MAKE_CALL:
363
364 return add_brk_on_nop(rec);
365
366 case FTRACE_UPDATE_MODIFY_CALL:
367 case FTRACE_UPDATE_MAKE_NOP:
368
369 return add_brk_on_call(rec, ftrace_addr);
370 }
371 return 0;
372}
373
374
375
376
377
378
379
380
381
382static int remove_breakpoint(struct dyn_ftrace *rec)
383{
384 unsigned char ins[MCOUNT_INSN_SIZE];
385 unsigned char brk = BREAKPOINT_INSTRUCTION;
386 const unsigned char *nop;
387 unsigned long ftrace_addr;
388 unsigned long ip = rec->ip;
389
390
391 if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
392 return -EFAULT;
393
394
395 if (ins[0] != brk)
396 return 0;
397
398 nop = ftrace_nop_replace();
399
400
401
402
403
404 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
405
406
407
408
409
410
411 ftrace_addr = ftrace_get_addr_new(rec);
412 nop = ftrace_call_replace(ip, ftrace_addr);
413
414 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
415 goto update;
416
417
418 ftrace_addr = ftrace_get_addr_curr(rec);
419 nop = ftrace_call_replace(ip, ftrace_addr);
420
421 ftrace_expected = nop;
422
423 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
424 return -EINVAL;
425 }
426
427 update:
428 return ftrace_write(ip, nop, 1);
429}
430
431static int add_update_code(unsigned long ip, unsigned const char *new)
432{
433
434 ip++;
435 new++;
436 return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
437}
438
439static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
440{
441 unsigned long ip = rec->ip;
442 unsigned const char *new;
443
444 new = ftrace_call_replace(ip, addr);
445 return add_update_code(ip, new);
446}
447
448static int add_update_nop(struct dyn_ftrace *rec)
449{
450 unsigned long ip = rec->ip;
451 unsigned const char *new;
452
453 new = ftrace_nop_replace();
454 return add_update_code(ip, new);
455}
456
457static int add_update(struct dyn_ftrace *rec, int enable)
458{
459 unsigned long ftrace_addr;
460 int ret;
461
462 ret = ftrace_test_record(rec, enable);
463
464 ftrace_addr = ftrace_get_addr_new(rec);
465
466 switch (ret) {
467 case FTRACE_UPDATE_IGNORE:
468 return 0;
469
470 case FTRACE_UPDATE_MODIFY_CALL:
471 case FTRACE_UPDATE_MAKE_CALL:
472
473 return add_update_call(rec, ftrace_addr);
474
475 case FTRACE_UPDATE_MAKE_NOP:
476
477 return add_update_nop(rec);
478 }
479
480 return 0;
481}
482
483static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
484{
485 unsigned long ip = rec->ip;
486 unsigned const char *new;
487
488 new = ftrace_call_replace(ip, addr);
489
490 return ftrace_write(ip, new, 1);
491}
492
493static int finish_update_nop(struct dyn_ftrace *rec)
494{
495 unsigned long ip = rec->ip;
496 unsigned const char *new;
497
498 new = ftrace_nop_replace();
499
500 return ftrace_write(ip, new, 1);
501}
502
503static int finish_update(struct dyn_ftrace *rec, int enable)
504{
505 unsigned long ftrace_addr;
506 int ret;
507
508 ret = ftrace_update_record(rec, enable);
509
510 ftrace_addr = ftrace_get_addr_new(rec);
511
512 switch (ret) {
513 case FTRACE_UPDATE_IGNORE:
514 return 0;
515
516 case FTRACE_UPDATE_MODIFY_CALL:
517 case FTRACE_UPDATE_MAKE_CALL:
518
519 return finish_update_call(rec, ftrace_addr);
520
521 case FTRACE_UPDATE_MAKE_NOP:
522
523 return finish_update_nop(rec);
524 }
525
526 return 0;
527}
528
529static void do_sync_core(void *data)
530{
531 sync_core();
532}
533
534static void run_sync(void)
535{
536 int enable_irqs;
537
538
539 if (num_online_cpus() == 1)
540 return;
541
542 enable_irqs = irqs_disabled();
543
544
545 if (enable_irqs)
546 local_irq_enable();
547 on_each_cpu(do_sync_core, NULL, 1);
548 if (enable_irqs)
549 local_irq_disable();
550}
551
552void ftrace_replace_code(int enable)
553{
554 struct ftrace_rec_iter *iter;
555 struct dyn_ftrace *rec;
556 const char *report = "adding breakpoints";
557 int count = 0;
558 int ret;
559
560 for_ftrace_rec_iter(iter) {
561 rec = ftrace_rec_iter_record(iter);
562
563 ret = add_breakpoints(rec, enable);
564 if (ret)
565 goto remove_breakpoints;
566 count++;
567 }
568
569 run_sync();
570
571 report = "updating code";
572 count = 0;
573
574 for_ftrace_rec_iter(iter) {
575 rec = ftrace_rec_iter_record(iter);
576
577 ret = add_update(rec, enable);
578 if (ret)
579 goto remove_breakpoints;
580 count++;
581 }
582
583 run_sync();
584
585 report = "removing breakpoints";
586 count = 0;
587
588 for_ftrace_rec_iter(iter) {
589 rec = ftrace_rec_iter_record(iter);
590
591 ret = finish_update(rec, enable);
592 if (ret)
593 goto remove_breakpoints;
594 count++;
595 }
596
597 run_sync();
598
599 return;
600
601 remove_breakpoints:
602 pr_warn("Failed on %s (%d):\n", report, count);
603 ftrace_bug(ret, rec);
604 for_ftrace_rec_iter(iter) {
605 rec = ftrace_rec_iter_record(iter);
606
607
608
609
610 if (remove_breakpoint(rec))
611 BUG();
612 }
613 run_sync();
614}
615
616static int
617ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
618 unsigned const char *new_code)
619{
620 int ret;
621
622 ret = add_break(ip, old_code);
623 if (ret)
624 goto out;
625
626 run_sync();
627
628 ret = add_update_code(ip, new_code);
629 if (ret)
630 goto fail_update;
631
632 run_sync();
633
634 ret = ftrace_write(ip, new_code, 1);
635
636
637
638
639 BUG_ON(ret);
640 out:
641 run_sync();
642 return ret;
643
644 fail_update:
645
646 if (ftrace_write(ip, old_code, 1))
647 BUG();
648 goto out;
649}
650
651void arch_ftrace_update_code(int command)
652{
653
654 atomic_inc(&modifying_ftrace_code);
655
656 ftrace_modify_all_code(command);
657
658 atomic_dec(&modifying_ftrace_code);
659}
660
661int __init ftrace_dyn_arch_init(void)
662{
663 return 0;
664}
665
666#if defined(CONFIG_X86_64) || defined(CONFIG_FUNCTION_GRAPH_TRACER)
667static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
668{
669 static union ftrace_code_union calc;
670
671
672 calc.e8 = 0xe9;
673 calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
674
675
676
677
678 return calc.code;
679}
680#endif
681
682
683#ifdef CONFIG_X86_64
684
685#ifdef CONFIG_MODULES
686#include <linux/moduleloader.h>
687
688static inline void *alloc_tramp(unsigned long size)
689{
690 return module_alloc(size);
691}
692static inline void tramp_free(void *tramp, int size)
693{
694 int npages = PAGE_ALIGN(size) >> PAGE_SHIFT;
695
696 set_memory_nx((unsigned long)tramp, npages);
697 set_memory_rw((unsigned long)tramp, npages);
698 module_memfree(tramp);
699}
700#else
701
702static inline void *alloc_tramp(unsigned long size)
703{
704 return NULL;
705}
706static inline void tramp_free(void *tramp, int size) { }
707#endif
708
709
710extern void ftrace_regs_caller_end(void);
711extern void ftrace_epilogue(void);
712extern void ftrace_caller_op_ptr(void);
713extern void ftrace_regs_caller_op_ptr(void);
714
715
716
717#define OP_REF_SIZE 7
718
719
720
721
722
723
724
725
726
727union ftrace_op_code_union {
728 char code[OP_REF_SIZE];
729 struct {
730 char op[3];
731 int offset;
732 } __attribute__((packed));
733};
734
735static unsigned long
736create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
737{
738 unsigned const char *jmp;
739 unsigned long start_offset;
740 unsigned long end_offset;
741 unsigned long op_offset;
742 unsigned long offset;
743 unsigned long size;
744 unsigned long ip;
745 unsigned long *ptr;
746 void *trampoline;
747
748 unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
749 union ftrace_op_code_union op_ptr;
750 int ret;
751
752 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
753 start_offset = (unsigned long)ftrace_regs_caller;
754 end_offset = (unsigned long)ftrace_regs_caller_end;
755 op_offset = (unsigned long)ftrace_regs_caller_op_ptr;
756 } else {
757 start_offset = (unsigned long)ftrace_caller;
758 end_offset = (unsigned long)ftrace_epilogue;
759 op_offset = (unsigned long)ftrace_caller_op_ptr;
760 }
761
762 size = end_offset - start_offset;
763
764
765
766
767
768
769 trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *));
770 if (!trampoline)
771 return 0;
772
773 *tramp_size = size + MCOUNT_INSN_SIZE + sizeof(void *);
774
775
776 ret = probe_kernel_read(trampoline, (void *)start_offset, size);
777 if (WARN_ON(ret < 0)) {
778 tramp_free(trampoline, *tramp_size);
779 return 0;
780 }
781
782 ip = (unsigned long)trampoline + size;
783
784
785 jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_epilogue);
786 memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE);
787
788
789
790
791
792
793
794
795
796 ptr = (unsigned long *)(trampoline + size + MCOUNT_INSN_SIZE);
797 *ptr = (unsigned long)ops;
798
799 op_offset -= start_offset;
800 memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE);
801
802
803 if (WARN_ON(memcmp(op_ptr.op, op_ref, 3) != 0)) {
804 tramp_free(trampoline, *tramp_size);
805 return 0;
806 }
807
808
809 offset = (unsigned long)ptr;
810 offset -= (unsigned long)trampoline + op_offset + OP_REF_SIZE;
811
812 op_ptr.offset = offset;
813
814
815 memcpy(trampoline + op_offset, &op_ptr, OP_REF_SIZE);
816
817
818 ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
819
820 return (unsigned long)trampoline;
821}
822
823static unsigned long calc_trampoline_call_offset(bool save_regs)
824{
825 unsigned long start_offset;
826 unsigned long call_offset;
827
828 if (save_regs) {
829 start_offset = (unsigned long)ftrace_regs_caller;
830 call_offset = (unsigned long)ftrace_regs_call;
831 } else {
832 start_offset = (unsigned long)ftrace_caller;
833 call_offset = (unsigned long)ftrace_call;
834 }
835
836 return call_offset - start_offset;
837}
838
839void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
840{
841 ftrace_func_t func;
842 unsigned char *new;
843 unsigned long offset;
844 unsigned long ip;
845 unsigned int size;
846 int ret, npages;
847
848 if (ops->trampoline) {
849
850
851
852
853 if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
854 return;
855 npages = PAGE_ALIGN(ops->trampoline_size) >> PAGE_SHIFT;
856 set_memory_rw(ops->trampoline, npages);
857 } else {
858 ops->trampoline = create_trampoline(ops, &size);
859 if (!ops->trampoline)
860 return;
861 ops->trampoline_size = size;
862 npages = PAGE_ALIGN(size) >> PAGE_SHIFT;
863 }
864
865 offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
866 ip = ops->trampoline + offset;
867
868 func = ftrace_ops_get_func(ops);
869
870
871 new = ftrace_call_replace(ip, (unsigned long)func);
872 ret = update_ftrace_func(ip, new);
873 set_memory_ro(ops->trampoline, npages);
874
875
876 WARN_ON(ret);
877}
878
879
880static void *addr_from_call(void *ptr)
881{
882 union ftrace_code_union calc;
883 int ret;
884
885 ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
886 if (WARN_ON_ONCE(ret < 0))
887 return NULL;
888
889
890 if (WARN_ON_ONCE(calc.e8 != 0xe8)) {
891 pr_warn("Expected e8, got %x\n", calc.e8);
892 return NULL;
893 }
894
895 return ptr + MCOUNT_INSN_SIZE + calc.offset;
896}
897
898void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
899 unsigned long frame_pointer);
900
901
902
903
904
905static void *static_tramp_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
906{
907 unsigned long offset;
908 bool save_regs = rec->flags & FTRACE_FL_REGS_EN;
909 void *ptr;
910
911 if (ops && ops->trampoline) {
912#ifdef CONFIG_FUNCTION_GRAPH_TRACER
913
914
915
916
917 if (ops->trampoline == FTRACE_GRAPH_ADDR)
918 return (void *)prepare_ftrace_return;
919#endif
920 return NULL;
921 }
922
923 offset = calc_trampoline_call_offset(save_regs);
924
925 if (save_regs)
926 ptr = (void *)FTRACE_REGS_ADDR + offset;
927 else
928 ptr = (void *)FTRACE_ADDR + offset;
929
930 return addr_from_call(ptr);
931}
932
933void *arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
934{
935 unsigned long offset;
936
937
938 if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
939 return static_tramp_func(ops, rec);
940
941 offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
942 return addr_from_call((void *)ops->trampoline + offset);
943}
944
945void arch_ftrace_trampoline_free(struct ftrace_ops *ops)
946{
947 if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
948 return;
949
950 tramp_free((void *)ops->trampoline, ops->trampoline_size);
951 ops->trampoline = 0;
952}
953
954#endif
955#endif
956
957#ifdef CONFIG_FUNCTION_GRAPH_TRACER
958
959#ifdef CONFIG_DYNAMIC_FTRACE
960extern void ftrace_graph_call(void);
961
962static int ftrace_mod_jmp(unsigned long ip, void *func)
963{
964 unsigned char *new;
965
966 new = ftrace_jmp_replace(ip, (unsigned long)func);
967
968 return update_ftrace_func(ip, new);
969}
970
971int ftrace_enable_ftrace_graph_caller(void)
972{
973 unsigned long ip = (unsigned long)(&ftrace_graph_call);
974
975 return ftrace_mod_jmp(ip, &ftrace_graph_caller);
976}
977
978int ftrace_disable_ftrace_graph_caller(void)
979{
980 unsigned long ip = (unsigned long)(&ftrace_graph_call);
981
982 return ftrace_mod_jmp(ip, &ftrace_stub);
983}
984
985#endif
986
987
988
989
990
991void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
992 unsigned long frame_pointer)
993{
994 unsigned long old;
995 int faulted;
996 struct ftrace_graph_ent trace;
997 unsigned long return_hooker = (unsigned long)
998 &return_to_handler;
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009 if (unlikely((long)__builtin_frame_address(0) >= 0))
1010 return;
1011
1012 if (unlikely(ftrace_graph_is_dead()))
1013 return;
1014
1015 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
1016 return;
1017
1018
1019
1020
1021
1022
1023 asm volatile(
1024 "1: " _ASM_MOV " (%[parent]), %[old]\n"
1025 "2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
1026 " movl $0, %[faulted]\n"
1027 "3:\n"
1028
1029 ".section .fixup, \"ax\"\n"
1030 "4: movl $1, %[faulted]\n"
1031 " jmp 3b\n"
1032 ".previous\n"
1033
1034 _ASM_EXTABLE(1b, 4b)
1035 _ASM_EXTABLE(2b, 4b)
1036
1037 : [old] "=&r" (old), [faulted] "=r" (faulted)
1038 : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
1039 : "memory"
1040 );
1041
1042 if (unlikely(faulted)) {
1043 ftrace_graph_stop();
1044 WARN_ON(1);
1045 return;
1046 }
1047
1048 trace.func = self_addr;
1049 trace.depth = current->curr_ret_stack + 1;
1050
1051
1052 if (!ftrace_graph_entry(&trace)) {
1053 *parent = old;
1054 return;
1055 }
1056
1057 if (ftrace_push_return_trace(old, self_addr, &trace.depth,
1058 frame_pointer, parent) == -EBUSY) {
1059 *parent = old;
1060 return;
1061 }
1062}
1063#endif
1064