1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#define pr_fmt(fmt) "kprobes: " fmt
22
23#include <linux/kprobes.h>
24#include <linux/hash.h>
25#include <linux/init.h>
26#include <linux/slab.h>
27#include <linux/stddef.h>
28#include <linux/export.h>
29#include <linux/moduleloader.h>
30#include <linux/kallsyms.h>
31#include <linux/freezer.h>
32#include <linux/seq_file.h>
33#include <linux/debugfs.h>
34#include <linux/sysctl.h>
35#include <linux/kdebug.h>
36#include <linux/memory.h>
37#include <linux/ftrace.h>
38#include <linux/cpu.h>
39#include <linux/jump_label.h>
40#include <linux/static_call.h>
41#include <linux/perf_event.h>
42
43#include <asm/sections.h>
44#include <asm/cacheflush.h>
45#include <asm/errno.h>
46#include <linux/uaccess.h>
47
48#define KPROBE_HASH_BITS 6
49#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
50
51
52static int kprobes_initialized;
53
54
55
56
57
58static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
59
60
61static bool kprobes_all_disarmed;
62
63
64static DEFINE_MUTEX(kprobe_mutex);
65static DEFINE_PER_CPU(struct kprobe *, kprobe_instance);
66
67kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
68 unsigned int __unused)
69{
70 return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
71}
72
73
74
75
76
77static LIST_HEAD(kprobe_blacklist);
78
79#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
80
81
82
83
84
85
86struct kprobe_insn_page {
87 struct list_head list;
88 kprobe_opcode_t *insns;
89 struct kprobe_insn_cache *cache;
90 int nused;
91 int ngarbage;
92 char slot_used[];
93};
94
95#define KPROBE_INSN_PAGE_SIZE(slots) \
96 (offsetof(struct kprobe_insn_page, slot_used) + \
97 (sizeof(char) * (slots)))
98
99static int slots_per_page(struct kprobe_insn_cache *c)
100{
101 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
102}
103
104enum kprobe_slot_state {
105 SLOT_CLEAN = 0,
106 SLOT_DIRTY = 1,
107 SLOT_USED = 2,
108};
109
110void __weak *alloc_insn_page(void)
111{
112
113
114
115
116
117
118 return module_alloc(PAGE_SIZE);
119}
120
121static void free_insn_page(void *page)
122{
123 module_memfree(page);
124}
125
126struct kprobe_insn_cache kprobe_insn_slots = {
127 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
128 .alloc = alloc_insn_page,
129 .free = free_insn_page,
130 .sym = KPROBE_INSN_PAGE_SYM,
131 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
132 .insn_size = MAX_INSN_SIZE,
133 .nr_garbage = 0,
134};
135static int collect_garbage_slots(struct kprobe_insn_cache *c);
136
137
138
139
140
141kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c)
142{
143 struct kprobe_insn_page *kip;
144 kprobe_opcode_t *slot = NULL;
145
146
147 mutex_lock(&c->mutex);
148 retry:
149 rcu_read_lock();
150 list_for_each_entry_rcu(kip, &c->pages, list) {
151 if (kip->nused < slots_per_page(c)) {
152 int i;
153
154 for (i = 0; i < slots_per_page(c); i++) {
155 if (kip->slot_used[i] == SLOT_CLEAN) {
156 kip->slot_used[i] = SLOT_USED;
157 kip->nused++;
158 slot = kip->insns + (i * c->insn_size);
159 rcu_read_unlock();
160 goto out;
161 }
162 }
163
164 kip->nused = slots_per_page(c);
165 WARN_ON(1);
166 }
167 }
168 rcu_read_unlock();
169
170
171 if (c->nr_garbage && collect_garbage_slots(c) == 0)
172 goto retry;
173
174
175 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
176 if (!kip)
177 goto out;
178
179 kip->insns = c->alloc();
180 if (!kip->insns) {
181 kfree(kip);
182 goto out;
183 }
184 INIT_LIST_HEAD(&kip->list);
185 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
186 kip->slot_used[0] = SLOT_USED;
187 kip->nused = 1;
188 kip->ngarbage = 0;
189 kip->cache = c;
190 list_add_rcu(&kip->list, &c->pages);
191 slot = kip->insns;
192
193
194 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns,
195 PAGE_SIZE, false, c->sym);
196out:
197 mutex_unlock(&c->mutex);
198 return slot;
199}
200
201
202static bool collect_one_slot(struct kprobe_insn_page *kip, int idx)
203{
204 kip->slot_used[idx] = SLOT_CLEAN;
205 kip->nused--;
206 if (kip->nused == 0) {
207
208
209
210
211
212
213 if (!list_is_singular(&kip->list)) {
214
215
216
217
218 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
219 (unsigned long)kip->insns, PAGE_SIZE, true,
220 kip->cache->sym);
221 list_del_rcu(&kip->list);
222 synchronize_rcu();
223 kip->cache->free(kip->insns);
224 kfree(kip);
225 }
226 return true;
227 }
228 return false;
229}
230
231static int collect_garbage_slots(struct kprobe_insn_cache *c)
232{
233 struct kprobe_insn_page *kip, *next;
234
235
236 synchronize_rcu();
237
238 list_for_each_entry_safe(kip, next, &c->pages, list) {
239 int i;
240
241 if (kip->ngarbage == 0)
242 continue;
243 kip->ngarbage = 0;
244 for (i = 0; i < slots_per_page(c); i++) {
245 if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
246 break;
247 }
248 }
249 c->nr_garbage = 0;
250 return 0;
251}
252
253void __free_insn_slot(struct kprobe_insn_cache *c,
254 kprobe_opcode_t *slot, int dirty)
255{
256 struct kprobe_insn_page *kip;
257 long idx;
258
259 mutex_lock(&c->mutex);
260 rcu_read_lock();
261 list_for_each_entry_rcu(kip, &c->pages, list) {
262 idx = ((long)slot - (long)kip->insns) /
263 (c->insn_size * sizeof(kprobe_opcode_t));
264 if (idx >= 0 && idx < slots_per_page(c))
265 goto out;
266 }
267
268 WARN_ON(1);
269 kip = NULL;
270out:
271 rcu_read_unlock();
272
273 if (kip) {
274
275 WARN_ON(kip->slot_used[idx] != SLOT_USED);
276 if (dirty) {
277 kip->slot_used[idx] = SLOT_DIRTY;
278 kip->ngarbage++;
279 if (++c->nr_garbage > slots_per_page(c))
280 collect_garbage_slots(c);
281 } else {
282 collect_one_slot(kip, idx);
283 }
284 }
285 mutex_unlock(&c->mutex);
286}
287
288
289
290
291
292
293bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
294{
295 struct kprobe_insn_page *kip;
296 bool ret = false;
297
298 rcu_read_lock();
299 list_for_each_entry_rcu(kip, &c->pages, list) {
300 if (addr >= (unsigned long)kip->insns &&
301 addr < (unsigned long)kip->insns + PAGE_SIZE) {
302 ret = true;
303 break;
304 }
305 }
306 rcu_read_unlock();
307
308 return ret;
309}
310
311int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum,
312 unsigned long *value, char *type, char *sym)
313{
314 struct kprobe_insn_page *kip;
315 int ret = -ERANGE;
316
317 rcu_read_lock();
318 list_for_each_entry_rcu(kip, &c->pages, list) {
319 if ((*symnum)--)
320 continue;
321 strscpy(sym, c->sym, KSYM_NAME_LEN);
322 *type = 't';
323 *value = (unsigned long)kip->insns;
324 ret = 0;
325 break;
326 }
327 rcu_read_unlock();
328
329 return ret;
330}
331
332#ifdef CONFIG_OPTPROBES
333void __weak *alloc_optinsn_page(void)
334{
335 return alloc_insn_page();
336}
337
338void __weak free_optinsn_page(void *page)
339{
340 free_insn_page(page);
341}
342
343
344struct kprobe_insn_cache kprobe_optinsn_slots = {
345 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
346 .alloc = alloc_optinsn_page,
347 .free = free_optinsn_page,
348 .sym = KPROBE_OPTINSN_PAGE_SYM,
349 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
350
351 .nr_garbage = 0,
352};
353#endif
354#endif
355
356
357static inline void set_kprobe_instance(struct kprobe *kp)
358{
359 __this_cpu_write(kprobe_instance, kp);
360}
361
362static inline void reset_kprobe_instance(void)
363{
364 __this_cpu_write(kprobe_instance, NULL);
365}
366
367
368
369
370
371
372
373struct kprobe *get_kprobe(void *addr)
374{
375 struct hlist_head *head;
376 struct kprobe *p;
377
378 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
379 hlist_for_each_entry_rcu(p, head, hlist,
380 lockdep_is_held(&kprobe_mutex)) {
381 if (p->addr == addr)
382 return p;
383 }
384
385 return NULL;
386}
387NOKPROBE_SYMBOL(get_kprobe);
388
389static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
390
391
392static inline bool kprobe_aggrprobe(struct kprobe *p)
393{
394 return p->pre_handler == aggr_pre_handler;
395}
396
397
398static inline bool kprobe_unused(struct kprobe *p)
399{
400 return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
401 list_empty(&p->list);
402}
403
404
405static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
406{
407 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
408 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
409}
410
411#ifdef CONFIG_OPTPROBES
412
413static bool kprobes_allow_optimization;
414
415
416
417
418
419void opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
420{
421 struct kprobe *kp;
422
423 list_for_each_entry_rcu(kp, &p->list, list) {
424 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
425 set_kprobe_instance(kp);
426 kp->pre_handler(kp, regs);
427 }
428 reset_kprobe_instance();
429 }
430}
431NOKPROBE_SYMBOL(opt_pre_handler);
432
433
434static void free_aggr_kprobe(struct kprobe *p)
435{
436 struct optimized_kprobe *op;
437
438 op = container_of(p, struct optimized_kprobe, kp);
439 arch_remove_optimized_kprobe(op);
440 arch_remove_kprobe(p);
441 kfree(op);
442}
443
444
445static inline int kprobe_optready(struct kprobe *p)
446{
447 struct optimized_kprobe *op;
448
449 if (kprobe_aggrprobe(p)) {
450 op = container_of(p, struct optimized_kprobe, kp);
451 return arch_prepared_optinsn(&op->optinsn);
452 }
453
454 return 0;
455}
456
457
458static inline bool kprobe_disarmed(struct kprobe *p)
459{
460 struct optimized_kprobe *op;
461
462
463 if (!kprobe_aggrprobe(p))
464 return kprobe_disabled(p);
465
466 op = container_of(p, struct optimized_kprobe, kp);
467
468 return kprobe_disabled(p) && list_empty(&op->list);
469}
470
471
472static bool kprobe_queued(struct kprobe *p)
473{
474 struct optimized_kprobe *op;
475
476 if (kprobe_aggrprobe(p)) {
477 op = container_of(p, struct optimized_kprobe, kp);
478 if (!list_empty(&op->list))
479 return true;
480 }
481 return false;
482}
483
484
485
486
487
488static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr)
489{
490 int i;
491 struct kprobe *p = NULL;
492 struct optimized_kprobe *op;
493
494
495 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++)
496 p = get_kprobe(addr - i);
497
498 if (p && kprobe_optready(p)) {
499 op = container_of(p, struct optimized_kprobe, kp);
500 if (arch_within_optimized_kprobe(op, addr))
501 return p;
502 }
503
504 return NULL;
505}
506
507
508static LIST_HEAD(optimizing_list);
509static LIST_HEAD(unoptimizing_list);
510static LIST_HEAD(freeing_list);
511
512static void kprobe_optimizer(struct work_struct *work);
513static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
514#define OPTIMIZE_DELAY 5
515
516
517
518
519
520static void do_optimize_kprobes(void)
521{
522 lockdep_assert_held(&text_mutex);
523
524
525
526
527
528
529
530
531
532
533 lockdep_assert_cpus_held();
534
535
536 if (kprobes_all_disarmed || !kprobes_allow_optimization ||
537 list_empty(&optimizing_list))
538 return;
539
540 arch_optimize_kprobes(&optimizing_list);
541}
542
543
544
545
546
547static void do_unoptimize_kprobes(void)
548{
549 struct optimized_kprobe *op, *tmp;
550
551 lockdep_assert_held(&text_mutex);
552
553 lockdep_assert_cpus_held();
554
555
556 if (list_empty(&unoptimizing_list))
557 return;
558
559 arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
560
561 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
562
563 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
564
565 if (kprobe_disabled(&op->kp))
566 arch_disarm_kprobe(&op->kp);
567 if (kprobe_unused(&op->kp)) {
568
569
570
571
572
573 hlist_del_rcu(&op->kp.hlist);
574 } else
575 list_del_init(&op->list);
576 }
577}
578
579
580static void do_free_cleaned_kprobes(void)
581{
582 struct optimized_kprobe *op, *tmp;
583
584 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
585 list_del_init(&op->list);
586 if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) {
587
588
589
590
591 continue;
592 }
593 free_aggr_kprobe(&op->kp);
594 }
595}
596
597
598static void kick_kprobe_optimizer(void)
599{
600 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
601}
602
603
604static void kprobe_optimizer(struct work_struct *work)
605{
606 mutex_lock(&kprobe_mutex);
607 cpus_read_lock();
608 mutex_lock(&text_mutex);
609
610
611
612
613
614 do_unoptimize_kprobes();
615
616
617
618
619
620
621
622
623
624
625 synchronize_rcu_tasks();
626
627
628 do_optimize_kprobes();
629
630
631 do_free_cleaned_kprobes();
632
633 mutex_unlock(&text_mutex);
634 cpus_read_unlock();
635
636
637 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
638 kick_kprobe_optimizer();
639
640 mutex_unlock(&kprobe_mutex);
641}
642
643
644void wait_for_kprobe_optimizer(void)
645{
646 mutex_lock(&kprobe_mutex);
647
648 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
649 mutex_unlock(&kprobe_mutex);
650
651
652 flush_delayed_work(&optimizing_work);
653
654 cpu_relax();
655
656 mutex_lock(&kprobe_mutex);
657 }
658
659 mutex_unlock(&kprobe_mutex);
660}
661
662static bool optprobe_queued_unopt(struct optimized_kprobe *op)
663{
664 struct optimized_kprobe *_op;
665
666 list_for_each_entry(_op, &unoptimizing_list, list) {
667 if (op == _op)
668 return true;
669 }
670
671 return false;
672}
673
674
675static void optimize_kprobe(struct kprobe *p)
676{
677 struct optimized_kprobe *op;
678
679
680 if (!kprobe_optready(p) || !kprobes_allow_optimization ||
681 (kprobe_disabled(p) || kprobes_all_disarmed))
682 return;
683
684
685 if (p->post_handler)
686 return;
687
688 op = container_of(p, struct optimized_kprobe, kp);
689
690
691 if (arch_check_optimized_kprobe(op) < 0)
692 return;
693
694
695 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) {
696 if (optprobe_queued_unopt(op)) {
697
698 list_del_init(&op->list);
699 }
700 return;
701 }
702 op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
703
704
705
706
707
708 if (WARN_ON_ONCE(!list_empty(&op->list)))
709 return;
710
711 list_add(&op->list, &optimizing_list);
712 kick_kprobe_optimizer();
713}
714
715
716static void force_unoptimize_kprobe(struct optimized_kprobe *op)
717{
718 lockdep_assert_cpus_held();
719 arch_unoptimize_kprobe(op);
720 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
721}
722
723
724static void unoptimize_kprobe(struct kprobe *p, bool force)
725{
726 struct optimized_kprobe *op;
727
728 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
729 return;
730
731 op = container_of(p, struct optimized_kprobe, kp);
732 if (!kprobe_optimized(p))
733 return;
734
735 if (!list_empty(&op->list)) {
736 if (optprobe_queued_unopt(op)) {
737
738 if (force) {
739
740
741
742
743 force_unoptimize_kprobe(op);
744 list_move(&op->list, &freeing_list);
745 }
746 } else {
747
748 list_del_init(&op->list);
749 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
750 }
751 return;
752 }
753
754
755 if (force) {
756
757 force_unoptimize_kprobe(op);
758 } else {
759 list_add(&op->list, &unoptimizing_list);
760 kick_kprobe_optimizer();
761 }
762}
763
764
765static int reuse_unused_kprobe(struct kprobe *ap)
766{
767 struct optimized_kprobe *op;
768
769
770
771
772
773 op = container_of(ap, struct optimized_kprobe, kp);
774 WARN_ON_ONCE(list_empty(&op->list));
775
776 ap->flags &= ~KPROBE_FLAG_DISABLED;
777
778 if (!kprobe_optready(ap))
779 return -EINVAL;
780
781 optimize_kprobe(ap);
782 return 0;
783}
784
785
786static void kill_optimized_kprobe(struct kprobe *p)
787{
788 struct optimized_kprobe *op;
789
790 op = container_of(p, struct optimized_kprobe, kp);
791 if (!list_empty(&op->list))
792
793 list_del_init(&op->list);
794 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
795
796 if (kprobe_unused(p)) {
797
798 list_add(&op->list, &freeing_list);
799
800
801
802
803
804 hlist_del_rcu(&op->kp.hlist);
805 }
806
807
808 arch_remove_optimized_kprobe(op);
809}
810
811static inline
812void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
813{
814 if (!kprobe_ftrace(p))
815 arch_prepare_optimized_kprobe(op, p);
816}
817
818
819static void prepare_optimized_kprobe(struct kprobe *p)
820{
821 struct optimized_kprobe *op;
822
823 op = container_of(p, struct optimized_kprobe, kp);
824 __prepare_optimized_kprobe(op, p);
825}
826
827
828static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
829{
830 struct optimized_kprobe *op;
831
832 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
833 if (!op)
834 return NULL;
835
836 INIT_LIST_HEAD(&op->list);
837 op->kp.addr = p->addr;
838 __prepare_optimized_kprobe(op, p);
839
840 return &op->kp;
841}
842
843static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
844
845
846
847
848
849static void try_to_optimize_kprobe(struct kprobe *p)
850{
851 struct kprobe *ap;
852 struct optimized_kprobe *op;
853
854
855 if (kprobe_ftrace(p))
856 return;
857
858
859 cpus_read_lock();
860 jump_label_lock();
861 mutex_lock(&text_mutex);
862
863 ap = alloc_aggr_kprobe(p);
864 if (!ap)
865 goto out;
866
867 op = container_of(ap, struct optimized_kprobe, kp);
868 if (!arch_prepared_optinsn(&op->optinsn)) {
869
870 arch_remove_optimized_kprobe(op);
871 kfree(op);
872 goto out;
873 }
874
875 init_aggr_kprobe(ap, p);
876 optimize_kprobe(ap);
877
878out:
879 mutex_unlock(&text_mutex);
880 jump_label_unlock();
881 cpus_read_unlock();
882}
883
884static void optimize_all_kprobes(void)
885{
886 struct hlist_head *head;
887 struct kprobe *p;
888 unsigned int i;
889
890 mutex_lock(&kprobe_mutex);
891
892 if (kprobes_allow_optimization)
893 goto out;
894
895 cpus_read_lock();
896 kprobes_allow_optimization = true;
897 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
898 head = &kprobe_table[i];
899 hlist_for_each_entry(p, head, hlist)
900 if (!kprobe_disabled(p))
901 optimize_kprobe(p);
902 }
903 cpus_read_unlock();
904 pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n");
905out:
906 mutex_unlock(&kprobe_mutex);
907}
908
909#ifdef CONFIG_SYSCTL
910static void unoptimize_all_kprobes(void)
911{
912 struct hlist_head *head;
913 struct kprobe *p;
914 unsigned int i;
915
916 mutex_lock(&kprobe_mutex);
917
918 if (!kprobes_allow_optimization) {
919 mutex_unlock(&kprobe_mutex);
920 return;
921 }
922
923 cpus_read_lock();
924 kprobes_allow_optimization = false;
925 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
926 head = &kprobe_table[i];
927 hlist_for_each_entry(p, head, hlist) {
928 if (!kprobe_disabled(p))
929 unoptimize_kprobe(p, false);
930 }
931 }
932 cpus_read_unlock();
933 mutex_unlock(&kprobe_mutex);
934
935
936 wait_for_kprobe_optimizer();
937 pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n");
938}
939
940static DEFINE_MUTEX(kprobe_sysctl_mutex);
941int sysctl_kprobes_optimization;
942int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
943 void *buffer, size_t *length,
944 loff_t *ppos)
945{
946 int ret;
947
948 mutex_lock(&kprobe_sysctl_mutex);
949 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
950 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
951
952 if (sysctl_kprobes_optimization)
953 optimize_all_kprobes();
954 else
955 unoptimize_all_kprobes();
956 mutex_unlock(&kprobe_sysctl_mutex);
957
958 return ret;
959}
960#endif
961
962
963static void __arm_kprobe(struct kprobe *p)
964{
965 struct kprobe *_p;
966
967 lockdep_assert_held(&text_mutex);
968
969
970 _p = get_optimized_kprobe(p->addr);
971 if (unlikely(_p))
972
973 unoptimize_kprobe(_p, true);
974
975 arch_arm_kprobe(p);
976 optimize_kprobe(p);
977}
978
979
980static void __disarm_kprobe(struct kprobe *p, bool reopt)
981{
982 struct kprobe *_p;
983
984 lockdep_assert_held(&text_mutex);
985
986
987 unoptimize_kprobe(p, kprobes_all_disarmed);
988
989 if (!kprobe_queued(p)) {
990 arch_disarm_kprobe(p);
991
992 _p = get_optimized_kprobe(p->addr);
993 if (unlikely(_p) && reopt)
994 optimize_kprobe(_p);
995 }
996
997
998
999
1000
1001
1002}
1003
1004#else
1005
1006#define optimize_kprobe(p) do {} while (0)
1007#define unoptimize_kprobe(p, f) do {} while (0)
1008#define kill_optimized_kprobe(p) do {} while (0)
1009#define prepare_optimized_kprobe(p) do {} while (0)
1010#define try_to_optimize_kprobe(p) do {} while (0)
1011#define __arm_kprobe(p) arch_arm_kprobe(p)
1012#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
1013#define kprobe_disarmed(p) kprobe_disabled(p)
1014#define wait_for_kprobe_optimizer() do {} while (0)
1015
1016static int reuse_unused_kprobe(struct kprobe *ap)
1017{
1018
1019
1020
1021
1022
1023
1024 WARN_ON_ONCE(1);
1025 return -EINVAL;
1026}
1027
1028static void free_aggr_kprobe(struct kprobe *p)
1029{
1030 arch_remove_kprobe(p);
1031 kfree(p);
1032}
1033
1034static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
1035{
1036 return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
1037}
1038#endif
1039
1040#ifdef CONFIG_KPROBES_ON_FTRACE
1041static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
1042 .func = kprobe_ftrace_handler,
1043 .flags = FTRACE_OPS_FL_SAVE_REGS,
1044};
1045
1046static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = {
1047 .func = kprobe_ftrace_handler,
1048 .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
1049};
1050
1051static int kprobe_ipmodify_enabled;
1052static int kprobe_ftrace_enabled;
1053
1054static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1055 int *cnt)
1056{
1057 int ret = 0;
1058
1059 lockdep_assert_held(&kprobe_mutex);
1060
1061 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0);
1062 if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret))
1063 return ret;
1064
1065 if (*cnt == 0) {
1066 ret = register_ftrace_function(ops);
1067 if (WARN(ret < 0, "Failed to register kprobe-ftrace (error %d)\n", ret))
1068 goto err_ftrace;
1069 }
1070
1071 (*cnt)++;
1072 return ret;
1073
1074err_ftrace:
1075
1076
1077
1078
1079 ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1080 return ret;
1081}
1082
1083static int arm_kprobe_ftrace(struct kprobe *p)
1084{
1085 bool ipmodify = (p->post_handler != NULL);
1086
1087 return __arm_kprobe_ftrace(p,
1088 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1089 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1090}
1091
1092static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1093 int *cnt)
1094{
1095 int ret = 0;
1096
1097 lockdep_assert_held(&kprobe_mutex);
1098
1099 if (*cnt == 1) {
1100 ret = unregister_ftrace_function(ops);
1101 if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret))
1102 return ret;
1103 }
1104
1105 (*cnt)--;
1106
1107 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1108 WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (error %d)\n",
1109 p->addr, ret);
1110 return ret;
1111}
1112
1113static int disarm_kprobe_ftrace(struct kprobe *p)
1114{
1115 bool ipmodify = (p->post_handler != NULL);
1116
1117 return __disarm_kprobe_ftrace(p,
1118 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1119 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1120}
1121#else
1122static inline int arm_kprobe_ftrace(struct kprobe *p)
1123{
1124 return -ENODEV;
1125}
1126
1127static inline int disarm_kprobe_ftrace(struct kprobe *p)
1128{
1129 return -ENODEV;
1130}
1131#endif
1132
1133static int prepare_kprobe(struct kprobe *p)
1134{
1135
1136 if (kprobe_ftrace(p))
1137 return arch_prepare_kprobe_ftrace(p);
1138
1139 return arch_prepare_kprobe(p);
1140}
1141
1142static int arm_kprobe(struct kprobe *kp)
1143{
1144 if (unlikely(kprobe_ftrace(kp)))
1145 return arm_kprobe_ftrace(kp);
1146
1147 cpus_read_lock();
1148 mutex_lock(&text_mutex);
1149 __arm_kprobe(kp);
1150 mutex_unlock(&text_mutex);
1151 cpus_read_unlock();
1152
1153 return 0;
1154}
1155
1156static int disarm_kprobe(struct kprobe *kp, bool reopt)
1157{
1158 if (unlikely(kprobe_ftrace(kp)))
1159 return disarm_kprobe_ftrace(kp);
1160
1161 cpus_read_lock();
1162 mutex_lock(&text_mutex);
1163 __disarm_kprobe(kp, reopt);
1164 mutex_unlock(&text_mutex);
1165 cpus_read_unlock();
1166
1167 return 0;
1168}
1169
1170
1171
1172
1173
1174static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
1175{
1176 struct kprobe *kp;
1177
1178 list_for_each_entry_rcu(kp, &p->list, list) {
1179 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
1180 set_kprobe_instance(kp);
1181 if (kp->pre_handler(kp, regs))
1182 return 1;
1183 }
1184 reset_kprobe_instance();
1185 }
1186 return 0;
1187}
1188NOKPROBE_SYMBOL(aggr_pre_handler);
1189
1190static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1191 unsigned long flags)
1192{
1193 struct kprobe *kp;
1194
1195 list_for_each_entry_rcu(kp, &p->list, list) {
1196 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
1197 set_kprobe_instance(kp);
1198 kp->post_handler(kp, regs, flags);
1199 reset_kprobe_instance();
1200 }
1201 }
1202}
1203NOKPROBE_SYMBOL(aggr_post_handler);
1204
1205
1206void kprobes_inc_nmissed_count(struct kprobe *p)
1207{
1208 struct kprobe *kp;
1209
1210 if (!kprobe_aggrprobe(p)) {
1211 p->nmissed++;
1212 } else {
1213 list_for_each_entry_rcu(kp, &p->list, list)
1214 kp->nmissed++;
1215 }
1216}
1217NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1218
1219static void free_rp_inst_rcu(struct rcu_head *head)
1220{
1221 struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu);
1222
1223 if (refcount_dec_and_test(&ri->rph->ref))
1224 kfree(ri->rph);
1225 kfree(ri);
1226}
1227NOKPROBE_SYMBOL(free_rp_inst_rcu);
1228
1229static void recycle_rp_inst(struct kretprobe_instance *ri)
1230{
1231 struct kretprobe *rp = get_kretprobe(ri);
1232
1233 if (likely(rp))
1234 freelist_add(&ri->freelist, &rp->freelist);
1235 else
1236 call_rcu(&ri->rcu, free_rp_inst_rcu);
1237}
1238NOKPROBE_SYMBOL(recycle_rp_inst);
1239
1240static struct kprobe kprobe_busy = {
1241 .addr = (void *) get_kprobe,
1242};
1243
1244void kprobe_busy_begin(void)
1245{
1246 struct kprobe_ctlblk *kcb;
1247
1248 preempt_disable();
1249 __this_cpu_write(current_kprobe, &kprobe_busy);
1250 kcb = get_kprobe_ctlblk();
1251 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
1252}
1253
1254void kprobe_busy_end(void)
1255{
1256 __this_cpu_write(current_kprobe, NULL);
1257 preempt_enable();
1258}
1259
1260
1261
1262
1263
1264
1265
1266void kprobe_flush_task(struct task_struct *tk)
1267{
1268 struct kretprobe_instance *ri;
1269 struct llist_node *node;
1270
1271
1272 if (unlikely(!kprobes_initialized))
1273 return;
1274
1275 kprobe_busy_begin();
1276
1277 node = __llist_del_all(&tk->kretprobe_instances);
1278 while (node) {
1279 ri = container_of(node, struct kretprobe_instance, llist);
1280 node = node->next;
1281
1282 recycle_rp_inst(ri);
1283 }
1284
1285 kprobe_busy_end();
1286}
1287NOKPROBE_SYMBOL(kprobe_flush_task);
1288
1289static inline void free_rp_inst(struct kretprobe *rp)
1290{
1291 struct kretprobe_instance *ri;
1292 struct freelist_node *node;
1293 int count = 0;
1294
1295 node = rp->freelist.head;
1296 while (node) {
1297 ri = container_of(node, struct kretprobe_instance, freelist);
1298 node = node->next;
1299
1300 kfree(ri);
1301 count++;
1302 }
1303
1304 if (refcount_sub_and_test(count, &rp->rph->ref)) {
1305 kfree(rp->rph);
1306 rp->rph = NULL;
1307 }
1308}
1309
1310
1311static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)
1312{
1313 if (p->post_handler)
1314 unoptimize_kprobe(ap, true);
1315
1316 list_add_rcu(&p->list, &ap->list);
1317 if (p->post_handler && !ap->post_handler)
1318 ap->post_handler = aggr_post_handler;
1319
1320 return 0;
1321}
1322
1323
1324
1325
1326
1327static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
1328{
1329
1330 copy_kprobe(p, ap);
1331 flush_insn_slot(ap);
1332 ap->addr = p->addr;
1333 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1334 ap->pre_handler = aggr_pre_handler;
1335
1336 if (p->post_handler && !kprobe_gone(p))
1337 ap->post_handler = aggr_post_handler;
1338
1339 INIT_LIST_HEAD(&ap->list);
1340 INIT_HLIST_NODE(&ap->hlist);
1341
1342 list_add_rcu(&p->list, &ap->list);
1343 hlist_replace_rcu(&p->hlist, &ap->hlist);
1344}
1345
1346
1347
1348
1349static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p)
1350{
1351 int ret = 0;
1352 struct kprobe *ap = orig_p;
1353
1354 cpus_read_lock();
1355
1356
1357 jump_label_lock();
1358 mutex_lock(&text_mutex);
1359
1360 if (!kprobe_aggrprobe(orig_p)) {
1361
1362 ap = alloc_aggr_kprobe(orig_p);
1363 if (!ap) {
1364 ret = -ENOMEM;
1365 goto out;
1366 }
1367 init_aggr_kprobe(ap, orig_p);
1368 } else if (kprobe_unused(ap)) {
1369
1370 ret = reuse_unused_kprobe(ap);
1371 if (ret)
1372 goto out;
1373 }
1374
1375 if (kprobe_gone(ap)) {
1376
1377
1378
1379
1380
1381
1382 ret = arch_prepare_kprobe(ap);
1383 if (ret)
1384
1385
1386
1387
1388
1389 goto out;
1390
1391
1392 prepare_optimized_kprobe(ap);
1393
1394
1395
1396
1397
1398 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1399 | KPROBE_FLAG_DISABLED;
1400 }
1401
1402
1403 copy_kprobe(ap, p);
1404 ret = add_new_kprobe(ap, p);
1405
1406out:
1407 mutex_unlock(&text_mutex);
1408 jump_label_unlock();
1409 cpus_read_unlock();
1410
1411 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1412 ap->flags &= ~KPROBE_FLAG_DISABLED;
1413 if (!kprobes_all_disarmed) {
1414
1415 ret = arm_kprobe(ap);
1416 if (ret) {
1417 ap->flags |= KPROBE_FLAG_DISABLED;
1418 list_del_rcu(&p->list);
1419 synchronize_rcu();
1420 }
1421 }
1422 }
1423 return ret;
1424}
1425
1426bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1427{
1428
1429 return addr >= (unsigned long)__kprobes_text_start &&
1430 addr < (unsigned long)__kprobes_text_end;
1431}
1432
1433static bool __within_kprobe_blacklist(unsigned long addr)
1434{
1435 struct kprobe_blacklist_entry *ent;
1436
1437 if (arch_within_kprobe_blacklist(addr))
1438 return true;
1439
1440
1441
1442
1443 list_for_each_entry(ent, &kprobe_blacklist, list) {
1444 if (addr >= ent->start_addr && addr < ent->end_addr)
1445 return true;
1446 }
1447 return false;
1448}
1449
1450bool within_kprobe_blacklist(unsigned long addr)
1451{
1452 char symname[KSYM_NAME_LEN], *p;
1453
1454 if (__within_kprobe_blacklist(addr))
1455 return true;
1456
1457
1458 if (!lookup_symbol_name(addr, symname)) {
1459 p = strchr(symname, '.');
1460 if (!p)
1461 return false;
1462 *p = '\0';
1463 addr = (unsigned long)kprobe_lookup_name(symname, 0);
1464 if (addr)
1465 return __within_kprobe_blacklist(addr);
1466 }
1467 return false;
1468}
1469
1470
1471
1472
1473
1474
1475
1476static kprobe_opcode_t *_kprobe_addr(kprobe_opcode_t *addr,
1477 const char *symbol_name, unsigned int offset)
1478{
1479 if ((symbol_name && addr) || (!symbol_name && !addr))
1480 goto invalid;
1481
1482 if (symbol_name) {
1483 addr = kprobe_lookup_name(symbol_name, offset);
1484 if (!addr)
1485 return ERR_PTR(-ENOENT);
1486 }
1487
1488 addr = (kprobe_opcode_t *)(((char *)addr) + offset);
1489 if (addr)
1490 return addr;
1491
1492invalid:
1493 return ERR_PTR(-EINVAL);
1494}
1495
1496static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
1497{
1498 return _kprobe_addr(p->addr, p->symbol_name, p->offset);
1499}
1500
1501
1502
1503
1504
1505static struct kprobe *__get_valid_kprobe(struct kprobe *p)
1506{
1507 struct kprobe *ap, *list_p;
1508
1509 lockdep_assert_held(&kprobe_mutex);
1510
1511 ap = get_kprobe(p->addr);
1512 if (unlikely(!ap))
1513 return NULL;
1514
1515 if (p != ap) {
1516 list_for_each_entry(list_p, &ap->list, list)
1517 if (list_p == p)
1518
1519 goto valid;
1520 return NULL;
1521 }
1522valid:
1523 return ap;
1524}
1525
1526
1527
1528
1529
1530static inline int warn_kprobe_rereg(struct kprobe *p)
1531{
1532 int ret = 0;
1533
1534 mutex_lock(&kprobe_mutex);
1535 if (WARN_ON_ONCE(__get_valid_kprobe(p)))
1536 ret = -EINVAL;
1537 mutex_unlock(&kprobe_mutex);
1538
1539 return ret;
1540}
1541
1542static int check_ftrace_location(struct kprobe *p)
1543{
1544 unsigned long ftrace_addr;
1545
1546 ftrace_addr = ftrace_location((unsigned long)p->addr);
1547 if (ftrace_addr) {
1548#ifdef CONFIG_KPROBES_ON_FTRACE
1549
1550 if ((unsigned long)p->addr != ftrace_addr)
1551 return -EILSEQ;
1552 p->flags |= KPROBE_FLAG_FTRACE;
1553#else
1554 return -EINVAL;
1555#endif
1556 }
1557 return 0;
1558}
1559
1560static int check_kprobe_address_safe(struct kprobe *p,
1561 struct module **probed_mod)
1562{
1563 int ret;
1564
1565 ret = check_ftrace_location(p);
1566 if (ret)
1567 return ret;
1568 jump_label_lock();
1569 preempt_disable();
1570
1571
1572 if (!kernel_text_address((unsigned long) p->addr) ||
1573 within_kprobe_blacklist((unsigned long) p->addr) ||
1574 jump_label_text_reserved(p->addr, p->addr) ||
1575 static_call_text_reserved(p->addr, p->addr) ||
1576 find_bug((unsigned long)p->addr)) {
1577 ret = -EINVAL;
1578 goto out;
1579 }
1580
1581
1582 *probed_mod = __module_text_address((unsigned long) p->addr);
1583 if (*probed_mod) {
1584
1585
1586
1587
1588 if (unlikely(!try_module_get(*probed_mod))) {
1589 ret = -ENOENT;
1590 goto out;
1591 }
1592
1593
1594
1595
1596
1597 if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1598 (*probed_mod)->state != MODULE_STATE_COMING) {
1599 module_put(*probed_mod);
1600 *probed_mod = NULL;
1601 ret = -ENOENT;
1602 }
1603 }
1604out:
1605 preempt_enable();
1606 jump_label_unlock();
1607
1608 return ret;
1609}
1610
1611int register_kprobe(struct kprobe *p)
1612{
1613 int ret;
1614 struct kprobe *old_p;
1615 struct module *probed_mod;
1616 kprobe_opcode_t *addr;
1617
1618
1619 addr = kprobe_addr(p);
1620 if (IS_ERR(addr))
1621 return PTR_ERR(addr);
1622 p->addr = addr;
1623
1624 ret = warn_kprobe_rereg(p);
1625 if (ret)
1626 return ret;
1627
1628
1629 p->flags &= KPROBE_FLAG_DISABLED;
1630 p->nmissed = 0;
1631 INIT_LIST_HEAD(&p->list);
1632
1633 ret = check_kprobe_address_safe(p, &probed_mod);
1634 if (ret)
1635 return ret;
1636
1637 mutex_lock(&kprobe_mutex);
1638
1639 old_p = get_kprobe(p->addr);
1640 if (old_p) {
1641
1642 ret = register_aggr_kprobe(old_p, p);
1643 goto out;
1644 }
1645
1646 cpus_read_lock();
1647
1648 mutex_lock(&text_mutex);
1649 ret = prepare_kprobe(p);
1650 mutex_unlock(&text_mutex);
1651 cpus_read_unlock();
1652 if (ret)
1653 goto out;
1654
1655 INIT_HLIST_NODE(&p->hlist);
1656 hlist_add_head_rcu(&p->hlist,
1657 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1658
1659 if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
1660 ret = arm_kprobe(p);
1661 if (ret) {
1662 hlist_del_rcu(&p->hlist);
1663 synchronize_rcu();
1664 goto out;
1665 }
1666 }
1667
1668
1669 try_to_optimize_kprobe(p);
1670out:
1671 mutex_unlock(&kprobe_mutex);
1672
1673 if (probed_mod)
1674 module_put(probed_mod);
1675
1676 return ret;
1677}
1678EXPORT_SYMBOL_GPL(register_kprobe);
1679
1680
1681static bool aggr_kprobe_disabled(struct kprobe *ap)
1682{
1683 struct kprobe *kp;
1684
1685 lockdep_assert_held(&kprobe_mutex);
1686
1687 list_for_each_entry(kp, &ap->list, list)
1688 if (!kprobe_disabled(kp))
1689
1690
1691
1692
1693 return false;
1694
1695 return true;
1696}
1697
1698static struct kprobe *__disable_kprobe(struct kprobe *p)
1699{
1700 struct kprobe *orig_p;
1701 int ret;
1702
1703 lockdep_assert_held(&kprobe_mutex);
1704
1705
1706 orig_p = __get_valid_kprobe(p);
1707 if (unlikely(orig_p == NULL))
1708 return ERR_PTR(-EINVAL);
1709
1710 if (!kprobe_disabled(p)) {
1711
1712 if (p != orig_p)
1713 p->flags |= KPROBE_FLAG_DISABLED;
1714
1715
1716 if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
1717
1718
1719
1720
1721
1722 if (!kprobes_all_disarmed) {
1723 ret = disarm_kprobe(orig_p, true);
1724 if (ret) {
1725 p->flags &= ~KPROBE_FLAG_DISABLED;
1726 return ERR_PTR(ret);
1727 }
1728 }
1729 orig_p->flags |= KPROBE_FLAG_DISABLED;
1730 }
1731 }
1732
1733 return orig_p;
1734}
1735
1736
1737
1738
1739static int __unregister_kprobe_top(struct kprobe *p)
1740{
1741 struct kprobe *ap, *list_p;
1742
1743
1744 ap = __disable_kprobe(p);
1745 if (IS_ERR(ap))
1746 return PTR_ERR(ap);
1747
1748 if (ap == p)
1749
1750
1751
1752
1753 goto disarmed;
1754
1755
1756 WARN_ON(!kprobe_aggrprobe(ap));
1757
1758 if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1759
1760
1761
1762
1763 goto disarmed;
1764 else {
1765
1766 if (p->post_handler && !kprobe_gone(p)) {
1767 list_for_each_entry(list_p, &ap->list, list) {
1768 if ((list_p != p) && (list_p->post_handler))
1769 goto noclean;
1770 }
1771 ap->post_handler = NULL;
1772 }
1773noclean:
1774
1775
1776
1777
1778 list_del_rcu(&p->list);
1779 if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1780
1781
1782
1783
1784 optimize_kprobe(ap);
1785 }
1786 return 0;
1787
1788disarmed:
1789 hlist_del_rcu(&ap->hlist);
1790 return 0;
1791}
1792
1793static void __unregister_kprobe_bottom(struct kprobe *p)
1794{
1795 struct kprobe *ap;
1796
1797 if (list_empty(&p->list))
1798
1799 arch_remove_kprobe(p);
1800 else if (list_is_singular(&p->list)) {
1801
1802 ap = list_entry(p->list.next, struct kprobe, list);
1803 list_del(&p->list);
1804 free_aggr_kprobe(ap);
1805 }
1806
1807}
1808
1809int register_kprobes(struct kprobe **kps, int num)
1810{
1811 int i, ret = 0;
1812
1813 if (num <= 0)
1814 return -EINVAL;
1815 for (i = 0; i < num; i++) {
1816 ret = register_kprobe(kps[i]);
1817 if (ret < 0) {
1818 if (i > 0)
1819 unregister_kprobes(kps, i);
1820 break;
1821 }
1822 }
1823 return ret;
1824}
1825EXPORT_SYMBOL_GPL(register_kprobes);
1826
1827void unregister_kprobe(struct kprobe *p)
1828{
1829 unregister_kprobes(&p, 1);
1830}
1831EXPORT_SYMBOL_GPL(unregister_kprobe);
1832
1833void unregister_kprobes(struct kprobe **kps, int num)
1834{
1835 int i;
1836
1837 if (num <= 0)
1838 return;
1839 mutex_lock(&kprobe_mutex);
1840 for (i = 0; i < num; i++)
1841 if (__unregister_kprobe_top(kps[i]) < 0)
1842 kps[i]->addr = NULL;
1843 mutex_unlock(&kprobe_mutex);
1844
1845 synchronize_rcu();
1846 for (i = 0; i < num; i++)
1847 if (kps[i]->addr)
1848 __unregister_kprobe_bottom(kps[i]);
1849}
1850EXPORT_SYMBOL_GPL(unregister_kprobes);
1851
1852int __weak kprobe_exceptions_notify(struct notifier_block *self,
1853 unsigned long val, void *data)
1854{
1855 return NOTIFY_DONE;
1856}
1857NOKPROBE_SYMBOL(kprobe_exceptions_notify);
1858
1859static struct notifier_block kprobe_exceptions_nb = {
1860 .notifier_call = kprobe_exceptions_notify,
1861 .priority = 0x7fffffff
1862};
1863
1864#ifdef CONFIG_KRETPROBES
1865
1866
1867static kprobe_opcode_t *__kretprobe_find_ret_addr(struct task_struct *tsk,
1868 struct llist_node **cur)
1869{
1870 struct kretprobe_instance *ri = NULL;
1871 struct llist_node *node = *cur;
1872
1873 if (!node)
1874 node = tsk->kretprobe_instances.first;
1875 else
1876 node = node->next;
1877
1878 while (node) {
1879 ri = container_of(node, struct kretprobe_instance, llist);
1880 if (ri->ret_addr != kretprobe_trampoline_addr()) {
1881 *cur = node;
1882 return ri->ret_addr;
1883 }
1884 node = node->next;
1885 }
1886 return NULL;
1887}
1888NOKPROBE_SYMBOL(__kretprobe_find_ret_addr);
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp,
1906 struct llist_node **cur)
1907{
1908 struct kretprobe_instance *ri = NULL;
1909 kprobe_opcode_t *ret;
1910
1911 if (WARN_ON_ONCE(!cur))
1912 return 0;
1913
1914 do {
1915 ret = __kretprobe_find_ret_addr(tsk, cur);
1916 if (!ret)
1917 break;
1918 ri = container_of(*cur, struct kretprobe_instance, llist);
1919 } while (ri->fp != fp);
1920
1921 return (unsigned long)ret;
1922}
1923NOKPROBE_SYMBOL(kretprobe_find_ret_addr);
1924
1925void __weak arch_kretprobe_fixup_return(struct pt_regs *regs,
1926 kprobe_opcode_t *correct_ret_addr)
1927{
1928
1929
1930
1931
1932}
1933
1934unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
1935 void *frame_pointer)
1936{
1937 kprobe_opcode_t *correct_ret_addr = NULL;
1938 struct kretprobe_instance *ri = NULL;
1939 struct llist_node *first, *node = NULL;
1940 struct kretprobe *rp;
1941
1942
1943 correct_ret_addr = __kretprobe_find_ret_addr(current, &node);
1944 if (!correct_ret_addr) {
1945 pr_err("kretprobe: Return address not found, not execute handler. Maybe there is a bug in the kernel.\n");
1946 BUG_ON(1);
1947 }
1948
1949
1950
1951
1952
1953
1954 instruction_pointer_set(regs, (unsigned long)correct_ret_addr);
1955
1956
1957 first = current->kretprobe_instances.first;
1958 while (first) {
1959 ri = container_of(first, struct kretprobe_instance, llist);
1960
1961 if (WARN_ON_ONCE(ri->fp != frame_pointer))
1962 break;
1963
1964 rp = get_kretprobe(ri);
1965 if (rp && rp->handler) {
1966 struct kprobe *prev = kprobe_running();
1967
1968 __this_cpu_write(current_kprobe, &rp->kp);
1969 ri->ret_addr = correct_ret_addr;
1970 rp->handler(ri, regs);
1971 __this_cpu_write(current_kprobe, prev);
1972 }
1973 if (first == node)
1974 break;
1975
1976 first = first->next;
1977 }
1978
1979 arch_kretprobe_fixup_return(regs, correct_ret_addr);
1980
1981
1982 first = current->kretprobe_instances.first;
1983 current->kretprobe_instances.first = node->next;
1984 node->next = NULL;
1985
1986
1987 while (first) {
1988 ri = container_of(first, struct kretprobe_instance, llist);
1989 first = first->next;
1990
1991 recycle_rp_inst(ri);
1992 }
1993
1994 return (unsigned long)correct_ret_addr;
1995}
1996NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
1997
1998
1999
2000
2001
2002static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2003{
2004 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2005 struct kretprobe_instance *ri;
2006 struct freelist_node *fn;
2007
2008 fn = freelist_try_get(&rp->freelist);
2009 if (!fn) {
2010 rp->nmissed++;
2011 return 0;
2012 }
2013
2014 ri = container_of(fn, struct kretprobe_instance, freelist);
2015
2016 if (rp->entry_handler && rp->entry_handler(ri, regs)) {
2017 freelist_add(&ri->freelist, &rp->freelist);
2018 return 0;
2019 }
2020
2021 arch_prepare_kretprobe(ri, regs);
2022
2023 __llist_add(&ri->llist, ¤t->kretprobe_instances);
2024
2025 return 0;
2026}
2027NOKPROBE_SYMBOL(pre_handler_kretprobe);
2028
2029bool __weak arch_kprobe_on_func_entry(unsigned long offset)
2030{
2031 return !offset;
2032}
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
2048{
2049 kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
2050
2051 if (IS_ERR(kp_addr))
2052 return PTR_ERR(kp_addr);
2053
2054 if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset))
2055 return -ENOENT;
2056
2057 if (!arch_kprobe_on_func_entry(offset))
2058 return -EINVAL;
2059
2060 return 0;
2061}
2062
2063int register_kretprobe(struct kretprobe *rp)
2064{
2065 int ret;
2066 struct kretprobe_instance *inst;
2067 int i;
2068 void *addr;
2069
2070 ret = kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset);
2071 if (ret)
2072 return ret;
2073
2074
2075 if (rp->kp.addr && warn_kprobe_rereg(&rp->kp))
2076 return -EINVAL;
2077
2078 if (kretprobe_blacklist_size) {
2079 addr = kprobe_addr(&rp->kp);
2080 if (IS_ERR(addr))
2081 return PTR_ERR(addr);
2082
2083 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2084 if (kretprobe_blacklist[i].addr == addr)
2085 return -EINVAL;
2086 }
2087 }
2088
2089 if (rp->data_size > KRETPROBE_MAX_DATA_SIZE)
2090 return -E2BIG;
2091
2092 rp->kp.pre_handler = pre_handler_kretprobe;
2093 rp->kp.post_handler = NULL;
2094
2095
2096 if (rp->maxactive <= 0) {
2097#ifdef CONFIG_PREEMPTION
2098 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
2099#else
2100 rp->maxactive = num_possible_cpus();
2101#endif
2102 }
2103 rp->freelist.head = NULL;
2104 rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL);
2105 if (!rp->rph)
2106 return -ENOMEM;
2107
2108 rp->rph->rp = rp;
2109 for (i = 0; i < rp->maxactive; i++) {
2110 inst = kzalloc(sizeof(struct kretprobe_instance) +
2111 rp->data_size, GFP_KERNEL);
2112 if (inst == NULL) {
2113 refcount_set(&rp->rph->ref, i);
2114 free_rp_inst(rp);
2115 return -ENOMEM;
2116 }
2117 inst->rph = rp->rph;
2118 freelist_add(&inst->freelist, &rp->freelist);
2119 }
2120 refcount_set(&rp->rph->ref, i);
2121
2122 rp->nmissed = 0;
2123
2124 ret = register_kprobe(&rp->kp);
2125 if (ret != 0)
2126 free_rp_inst(rp);
2127 return ret;
2128}
2129EXPORT_SYMBOL_GPL(register_kretprobe);
2130
2131int register_kretprobes(struct kretprobe **rps, int num)
2132{
2133 int ret = 0, i;
2134
2135 if (num <= 0)
2136 return -EINVAL;
2137 for (i = 0; i < num; i++) {
2138 ret = register_kretprobe(rps[i]);
2139 if (ret < 0) {
2140 if (i > 0)
2141 unregister_kretprobes(rps, i);
2142 break;
2143 }
2144 }
2145 return ret;
2146}
2147EXPORT_SYMBOL_GPL(register_kretprobes);
2148
2149void unregister_kretprobe(struct kretprobe *rp)
2150{
2151 unregister_kretprobes(&rp, 1);
2152}
2153EXPORT_SYMBOL_GPL(unregister_kretprobe);
2154
2155void unregister_kretprobes(struct kretprobe **rps, int num)
2156{
2157 int i;
2158
2159 if (num <= 0)
2160 return;
2161 mutex_lock(&kprobe_mutex);
2162 for (i = 0; i < num; i++) {
2163 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
2164 rps[i]->kp.addr = NULL;
2165 rps[i]->rph->rp = NULL;
2166 }
2167 mutex_unlock(&kprobe_mutex);
2168
2169 synchronize_rcu();
2170 for (i = 0; i < num; i++) {
2171 if (rps[i]->kp.addr) {
2172 __unregister_kprobe_bottom(&rps[i]->kp);
2173 free_rp_inst(rps[i]);
2174 }
2175 }
2176}
2177EXPORT_SYMBOL_GPL(unregister_kretprobes);
2178
2179#else
2180int register_kretprobe(struct kretprobe *rp)
2181{
2182 return -EOPNOTSUPP;
2183}
2184EXPORT_SYMBOL_GPL(register_kretprobe);
2185
2186int register_kretprobes(struct kretprobe **rps, int num)
2187{
2188 return -EOPNOTSUPP;
2189}
2190EXPORT_SYMBOL_GPL(register_kretprobes);
2191
2192void unregister_kretprobe(struct kretprobe *rp)
2193{
2194}
2195EXPORT_SYMBOL_GPL(unregister_kretprobe);
2196
2197void unregister_kretprobes(struct kretprobe **rps, int num)
2198{
2199}
2200EXPORT_SYMBOL_GPL(unregister_kretprobes);
2201
2202static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2203{
2204 return 0;
2205}
2206NOKPROBE_SYMBOL(pre_handler_kretprobe);
2207
2208#endif
2209
2210
2211static void kill_kprobe(struct kprobe *p)
2212{
2213 struct kprobe *kp;
2214
2215 lockdep_assert_held(&kprobe_mutex);
2216
2217 p->flags |= KPROBE_FLAG_GONE;
2218 if (kprobe_aggrprobe(p)) {
2219
2220
2221
2222
2223 list_for_each_entry(kp, &p->list, list)
2224 kp->flags |= KPROBE_FLAG_GONE;
2225 p->post_handler = NULL;
2226 kill_optimized_kprobe(p);
2227 }
2228
2229
2230
2231
2232 arch_remove_kprobe(p);
2233
2234
2235
2236
2237
2238
2239 if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed)
2240 disarm_kprobe_ftrace(p);
2241}
2242
2243
2244int disable_kprobe(struct kprobe *kp)
2245{
2246 int ret = 0;
2247 struct kprobe *p;
2248
2249 mutex_lock(&kprobe_mutex);
2250
2251
2252 p = __disable_kprobe(kp);
2253 if (IS_ERR(p))
2254 ret = PTR_ERR(p);
2255
2256 mutex_unlock(&kprobe_mutex);
2257 return ret;
2258}
2259EXPORT_SYMBOL_GPL(disable_kprobe);
2260
2261
2262int enable_kprobe(struct kprobe *kp)
2263{
2264 int ret = 0;
2265 struct kprobe *p;
2266
2267 mutex_lock(&kprobe_mutex);
2268
2269
2270 p = __get_valid_kprobe(kp);
2271 if (unlikely(p == NULL)) {
2272 ret = -EINVAL;
2273 goto out;
2274 }
2275
2276 if (kprobe_gone(kp)) {
2277
2278 ret = -EINVAL;
2279 goto out;
2280 }
2281
2282 if (p != kp)
2283 kp->flags &= ~KPROBE_FLAG_DISABLED;
2284
2285 if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2286 p->flags &= ~KPROBE_FLAG_DISABLED;
2287 ret = arm_kprobe(p);
2288 if (ret)
2289 p->flags |= KPROBE_FLAG_DISABLED;
2290 }
2291out:
2292 mutex_unlock(&kprobe_mutex);
2293 return ret;
2294}
2295EXPORT_SYMBOL_GPL(enable_kprobe);
2296
2297
2298void dump_kprobe(struct kprobe *kp)
2299{
2300 pr_err("Dump kprobe:\n.symbol_name = %s, .offset = %x, .addr = %pS\n",
2301 kp->symbol_name, kp->offset, kp->addr);
2302}
2303NOKPROBE_SYMBOL(dump_kprobe);
2304
2305int kprobe_add_ksym_blacklist(unsigned long entry)
2306{
2307 struct kprobe_blacklist_entry *ent;
2308 unsigned long offset = 0, size = 0;
2309
2310 if (!kernel_text_address(entry) ||
2311 !kallsyms_lookup_size_offset(entry, &size, &offset))
2312 return -EINVAL;
2313
2314 ent = kmalloc(sizeof(*ent), GFP_KERNEL);
2315 if (!ent)
2316 return -ENOMEM;
2317 ent->start_addr = entry;
2318 ent->end_addr = entry + size;
2319 INIT_LIST_HEAD(&ent->list);
2320 list_add_tail(&ent->list, &kprobe_blacklist);
2321
2322 return (int)size;
2323}
2324
2325
2326int kprobe_add_area_blacklist(unsigned long start, unsigned long end)
2327{
2328 unsigned long entry;
2329 int ret = 0;
2330
2331 for (entry = start; entry < end; entry += ret) {
2332 ret = kprobe_add_ksym_blacklist(entry);
2333 if (ret < 0)
2334 return ret;
2335 if (ret == 0)
2336 ret = 1;
2337 }
2338 return 0;
2339}
2340
2341
2342static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end)
2343{
2344 struct kprobe_blacklist_entry *ent, *n;
2345
2346 list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) {
2347 if (ent->start_addr < start || ent->start_addr >= end)
2348 continue;
2349 list_del(&ent->list);
2350 kfree(ent);
2351 }
2352}
2353
2354static void kprobe_remove_ksym_blacklist(unsigned long entry)
2355{
2356 kprobe_remove_area_blacklist(entry, entry + 1);
2357}
2358
2359int __weak arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value,
2360 char *type, char *sym)
2361{
2362 return -ERANGE;
2363}
2364
2365int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
2366 char *sym)
2367{
2368#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
2369 if (!kprobe_cache_get_kallsym(&kprobe_insn_slots, &symnum, value, type, sym))
2370 return 0;
2371#ifdef CONFIG_OPTPROBES
2372 if (!kprobe_cache_get_kallsym(&kprobe_optinsn_slots, &symnum, value, type, sym))
2373 return 0;
2374#endif
2375#endif
2376 if (!arch_kprobe_get_kallsym(&symnum, value, type, sym))
2377 return 0;
2378 return -ERANGE;
2379}
2380
2381int __init __weak arch_populate_kprobe_blacklist(void)
2382{
2383 return 0;
2384}
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394static int __init populate_kprobe_blacklist(unsigned long *start,
2395 unsigned long *end)
2396{
2397 unsigned long entry;
2398 unsigned long *iter;
2399 int ret;
2400
2401 for (iter = start; iter < end; iter++) {
2402 entry = (unsigned long)dereference_symbol_descriptor((void *)*iter);
2403 ret = kprobe_add_ksym_blacklist(entry);
2404 if (ret == -EINVAL)
2405 continue;
2406 if (ret < 0)
2407 return ret;
2408 }
2409
2410
2411 ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start,
2412 (unsigned long)__kprobes_text_end);
2413 if (ret)
2414 return ret;
2415
2416
2417 ret = kprobe_add_area_blacklist((unsigned long)__noinstr_text_start,
2418 (unsigned long)__noinstr_text_end);
2419
2420 return ret ? : arch_populate_kprobe_blacklist();
2421}
2422
2423static void add_module_kprobe_blacklist(struct module *mod)
2424{
2425 unsigned long start, end;
2426 int i;
2427
2428 if (mod->kprobe_blacklist) {
2429 for (i = 0; i < mod->num_kprobe_blacklist; i++)
2430 kprobe_add_ksym_blacklist(mod->kprobe_blacklist[i]);
2431 }
2432
2433 start = (unsigned long)mod->kprobes_text_start;
2434 if (start) {
2435 end = start + mod->kprobes_text_size;
2436 kprobe_add_area_blacklist(start, end);
2437 }
2438
2439 start = (unsigned long)mod->noinstr_text_start;
2440 if (start) {
2441 end = start + mod->noinstr_text_size;
2442 kprobe_add_area_blacklist(start, end);
2443 }
2444}
2445
2446static void remove_module_kprobe_blacklist(struct module *mod)
2447{
2448 unsigned long start, end;
2449 int i;
2450
2451 if (mod->kprobe_blacklist) {
2452 for (i = 0; i < mod->num_kprobe_blacklist; i++)
2453 kprobe_remove_ksym_blacklist(mod->kprobe_blacklist[i]);
2454 }
2455
2456 start = (unsigned long)mod->kprobes_text_start;
2457 if (start) {
2458 end = start + mod->kprobes_text_size;
2459 kprobe_remove_area_blacklist(start, end);
2460 }
2461
2462 start = (unsigned long)mod->noinstr_text_start;
2463 if (start) {
2464 end = start + mod->noinstr_text_size;
2465 kprobe_remove_area_blacklist(start, end);
2466 }
2467}
2468
2469
2470static int kprobes_module_callback(struct notifier_block *nb,
2471 unsigned long val, void *data)
2472{
2473 struct module *mod = data;
2474 struct hlist_head *head;
2475 struct kprobe *p;
2476 unsigned int i;
2477 int checkcore = (val == MODULE_STATE_GOING);
2478
2479 if (val == MODULE_STATE_COMING) {
2480 mutex_lock(&kprobe_mutex);
2481 add_module_kprobe_blacklist(mod);
2482 mutex_unlock(&kprobe_mutex);
2483 }
2484 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
2485 return NOTIFY_DONE;
2486
2487
2488
2489
2490
2491
2492
2493 mutex_lock(&kprobe_mutex);
2494 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2495 head = &kprobe_table[i];
2496 hlist_for_each_entry(p, head, hlist)
2497 if (within_module_init((unsigned long)p->addr, mod) ||
2498 (checkcore &&
2499 within_module_core((unsigned long)p->addr, mod))) {
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511 kill_kprobe(p);
2512 }
2513 }
2514 if (val == MODULE_STATE_GOING)
2515 remove_module_kprobe_blacklist(mod);
2516 mutex_unlock(&kprobe_mutex);
2517 return NOTIFY_DONE;
2518}
2519
2520static struct notifier_block kprobe_module_nb = {
2521 .notifier_call = kprobes_module_callback,
2522 .priority = 0
2523};
2524
2525void kprobe_free_init_mem(void)
2526{
2527 void *start = (void *)(&__init_begin);
2528 void *end = (void *)(&__init_end);
2529 struct hlist_head *head;
2530 struct kprobe *p;
2531 int i;
2532
2533 mutex_lock(&kprobe_mutex);
2534
2535
2536 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2537 head = &kprobe_table[i];
2538 hlist_for_each_entry(p, head, hlist) {
2539 if (start <= (void *)p->addr && (void *)p->addr < end)
2540 kill_kprobe(p);
2541 }
2542 }
2543
2544 mutex_unlock(&kprobe_mutex);
2545}
2546
2547static int __init init_kprobes(void)
2548{
2549 int i, err = 0;
2550
2551
2552
2553 for (i = 0; i < KPROBE_TABLE_SIZE; i++)
2554 INIT_HLIST_HEAD(&kprobe_table[i]);
2555
2556 err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2557 __stop_kprobe_blacklist);
2558 if (err)
2559 pr_err("Failed to populate blacklist (error %d), kprobes not restricted, be careful using them!\n", err);
2560
2561 if (kretprobe_blacklist_size) {
2562
2563 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2564 kretprobe_blacklist[i].addr =
2565 kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
2566 if (!kretprobe_blacklist[i].addr)
2567 pr_err("Failed to lookup symbol '%s' for kretprobe blacklist. Maybe the target function is removed or renamed.\n",
2568 kretprobe_blacklist[i].name);
2569 }
2570 }
2571
2572
2573 kprobes_all_disarmed = false;
2574
2575#if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2576
2577 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2578#endif
2579
2580 err = arch_init_kprobes();
2581 if (!err)
2582 err = register_die_notifier(&kprobe_exceptions_nb);
2583 if (!err)
2584 err = register_module_notifier(&kprobe_module_nb);
2585
2586 kprobes_initialized = (err == 0);
2587 return err;
2588}
2589early_initcall(init_kprobes);
2590
2591#if defined(CONFIG_OPTPROBES)
2592static int __init init_optprobes(void)
2593{
2594
2595
2596
2597
2598
2599 optimize_all_kprobes();
2600
2601 return 0;
2602}
2603subsys_initcall(init_optprobes);
2604#endif
2605
2606#ifdef CONFIG_DEBUG_FS
2607static void report_probe(struct seq_file *pi, struct kprobe *p,
2608 const char *sym, int offset, char *modname, struct kprobe *pp)
2609{
2610 char *kprobe_type;
2611 void *addr = p->addr;
2612
2613 if (p->pre_handler == pre_handler_kretprobe)
2614 kprobe_type = "r";
2615 else
2616 kprobe_type = "k";
2617
2618 if (!kallsyms_show_value(pi->file->f_cred))
2619 addr = NULL;
2620
2621 if (sym)
2622 seq_printf(pi, "%px %s %s+0x%x %s ",
2623 addr, kprobe_type, sym, offset,
2624 (modname ? modname : " "));
2625 else
2626 seq_printf(pi, "%px %s %pS ",
2627 addr, kprobe_type, p->addr);
2628
2629 if (!pp)
2630 pp = p;
2631 seq_printf(pi, "%s%s%s%s\n",
2632 (kprobe_gone(p) ? "[GONE]" : ""),
2633 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
2634 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2635 (kprobe_ftrace(pp) ? "[FTRACE]" : ""));
2636}
2637
2638static void *kprobe_seq_start(struct seq_file *f, loff_t *pos)
2639{
2640 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2641}
2642
2643static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
2644{
2645 (*pos)++;
2646 if (*pos >= KPROBE_TABLE_SIZE)
2647 return NULL;
2648 return pos;
2649}
2650
2651static void kprobe_seq_stop(struct seq_file *f, void *v)
2652{
2653
2654}
2655
2656static int show_kprobe_addr(struct seq_file *pi, void *v)
2657{
2658 struct hlist_head *head;
2659 struct kprobe *p, *kp;
2660 const char *sym = NULL;
2661 unsigned int i = *(loff_t *) v;
2662 unsigned long offset = 0;
2663 char *modname, namebuf[KSYM_NAME_LEN];
2664
2665 head = &kprobe_table[i];
2666 preempt_disable();
2667 hlist_for_each_entry_rcu(p, head, hlist) {
2668 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
2669 &offset, &modname, namebuf);
2670 if (kprobe_aggrprobe(p)) {
2671 list_for_each_entry_rcu(kp, &p->list, list)
2672 report_probe(pi, kp, sym, offset, modname, p);
2673 } else
2674 report_probe(pi, p, sym, offset, modname, NULL);
2675 }
2676 preempt_enable();
2677 return 0;
2678}
2679
2680static const struct seq_operations kprobes_sops = {
2681 .start = kprobe_seq_start,
2682 .next = kprobe_seq_next,
2683 .stop = kprobe_seq_stop,
2684 .show = show_kprobe_addr
2685};
2686
2687DEFINE_SEQ_ATTRIBUTE(kprobes);
2688
2689
2690static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
2691{
2692 mutex_lock(&kprobe_mutex);
2693 return seq_list_start(&kprobe_blacklist, *pos);
2694}
2695
2696static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos)
2697{
2698 return seq_list_next(v, &kprobe_blacklist, pos);
2699}
2700
2701static int kprobe_blacklist_seq_show(struct seq_file *m, void *v)
2702{
2703 struct kprobe_blacklist_entry *ent =
2704 list_entry(v, struct kprobe_blacklist_entry, list);
2705
2706
2707
2708
2709
2710 if (!kallsyms_show_value(m->file->f_cred))
2711 seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL,
2712 (void *)ent->start_addr);
2713 else
2714 seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr,
2715 (void *)ent->end_addr, (void *)ent->start_addr);
2716 return 0;
2717}
2718
2719static void kprobe_blacklist_seq_stop(struct seq_file *f, void *v)
2720{
2721 mutex_unlock(&kprobe_mutex);
2722}
2723
2724static const struct seq_operations kprobe_blacklist_sops = {
2725 .start = kprobe_blacklist_seq_start,
2726 .next = kprobe_blacklist_seq_next,
2727 .stop = kprobe_blacklist_seq_stop,
2728 .show = kprobe_blacklist_seq_show,
2729};
2730DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist);
2731
2732static int arm_all_kprobes(void)
2733{
2734 struct hlist_head *head;
2735 struct kprobe *p;
2736 unsigned int i, total = 0, errors = 0;
2737 int err, ret = 0;
2738
2739 mutex_lock(&kprobe_mutex);
2740
2741
2742 if (!kprobes_all_disarmed)
2743 goto already_enabled;
2744
2745
2746
2747
2748
2749
2750 kprobes_all_disarmed = false;
2751
2752 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2753 head = &kprobe_table[i];
2754
2755 hlist_for_each_entry(p, head, hlist) {
2756 if (!kprobe_disabled(p)) {
2757 err = arm_kprobe(p);
2758 if (err) {
2759 errors++;
2760 ret = err;
2761 }
2762 total++;
2763 }
2764 }
2765 }
2766
2767 if (errors)
2768 pr_warn("Kprobes globally enabled, but failed to enable %d out of %d probes. Please check which kprobes are kept disabled via debugfs.\n",
2769 errors, total);
2770 else
2771 pr_info("Kprobes globally enabled\n");
2772
2773already_enabled:
2774 mutex_unlock(&kprobe_mutex);
2775 return ret;
2776}
2777
2778static int disarm_all_kprobes(void)
2779{
2780 struct hlist_head *head;
2781 struct kprobe *p;
2782 unsigned int i, total = 0, errors = 0;
2783 int err, ret = 0;
2784
2785 mutex_lock(&kprobe_mutex);
2786
2787
2788 if (kprobes_all_disarmed) {
2789 mutex_unlock(&kprobe_mutex);
2790 return 0;
2791 }
2792
2793 kprobes_all_disarmed = true;
2794
2795 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2796 head = &kprobe_table[i];
2797
2798 hlist_for_each_entry(p, head, hlist) {
2799 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
2800 err = disarm_kprobe(p, false);
2801 if (err) {
2802 errors++;
2803 ret = err;
2804 }
2805 total++;
2806 }
2807 }
2808 }
2809
2810 if (errors)
2811 pr_warn("Kprobes globally disabled, but failed to disable %d out of %d probes. Please check which kprobes are kept enabled via debugfs.\n",
2812 errors, total);
2813 else
2814 pr_info("Kprobes globally disabled\n");
2815
2816 mutex_unlock(&kprobe_mutex);
2817
2818
2819 wait_for_kprobe_optimizer();
2820
2821 return ret;
2822}
2823
2824
2825
2826
2827
2828
2829static ssize_t read_enabled_file_bool(struct file *file,
2830 char __user *user_buf, size_t count, loff_t *ppos)
2831{
2832 char buf[3];
2833
2834 if (!kprobes_all_disarmed)
2835 buf[0] = '1';
2836 else
2837 buf[0] = '0';
2838 buf[1] = '\n';
2839 buf[2] = 0x00;
2840 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2841}
2842
2843static ssize_t write_enabled_file_bool(struct file *file,
2844 const char __user *user_buf, size_t count, loff_t *ppos)
2845{
2846 bool enable;
2847 int ret;
2848
2849 ret = kstrtobool_from_user(user_buf, count, &enable);
2850 if (ret)
2851 return ret;
2852
2853 ret = enable ? arm_all_kprobes() : disarm_all_kprobes();
2854 if (ret)
2855 return ret;
2856
2857 return count;
2858}
2859
2860static const struct file_operations fops_kp = {
2861 .read = read_enabled_file_bool,
2862 .write = write_enabled_file_bool,
2863 .llseek = default_llseek,
2864};
2865
2866static int __init debugfs_kprobe_init(void)
2867{
2868 struct dentry *dir;
2869
2870 dir = debugfs_create_dir("kprobes", NULL);
2871
2872 debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops);
2873
2874 debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp);
2875
2876 debugfs_create_file("blacklist", 0400, dir, NULL,
2877 &kprobe_blacklist_fops);
2878
2879 return 0;
2880}
2881
2882late_initcall(debugfs_kprobe_init);
2883#endif
2884