1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/kprobes.h>
22#include <linux/hash.h>
23#include <linux/init.h>
24#include <linux/slab.h>
25#include <linux/stddef.h>
26#include <linux/export.h>
27#include <linux/moduleloader.h>
28#include <linux/kallsyms.h>
29#include <linux/freezer.h>
30#include <linux/seq_file.h>
31#include <linux/debugfs.h>
32#include <linux/sysctl.h>
33#include <linux/kdebug.h>
34#include <linux/memory.h>
35#include <linux/ftrace.h>
36#include <linux/cpu.h>
37#include <linux/jump_label.h>
38
39#include <asm/sections.h>
40#include <asm/cacheflush.h>
41#include <asm/errno.h>
42#include <linux/uaccess.h>
43
44#define KPROBE_HASH_BITS 6
45#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
46
47
48static int kprobes_initialized;
49static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
50static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
51
52
53static bool kprobes_all_disarmed;
54
55
56static DEFINE_MUTEX(kprobe_mutex);
57static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
58static struct {
59 raw_spinlock_t lock ____cacheline_aligned_in_smp;
60} kretprobe_table_locks[KPROBE_TABLE_SIZE];
61
62kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
63 unsigned int __unused)
64{
65 return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
66}
67
68static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
69{
70 return &(kretprobe_table_locks[hash].lock);
71}
72
73
74static LIST_HEAD(kprobe_blacklist);
75
76#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
77
78
79
80
81
82
83struct kprobe_insn_page {
84 struct list_head list;
85 kprobe_opcode_t *insns;
86 struct kprobe_insn_cache *cache;
87 int nused;
88 int ngarbage;
89 char slot_used[];
90};
91
92#define KPROBE_INSN_PAGE_SIZE(slots) \
93 (offsetof(struct kprobe_insn_page, slot_used) + \
94 (sizeof(char) * (slots)))
95
96static int slots_per_page(struct kprobe_insn_cache *c)
97{
98 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
99}
100
101enum kprobe_slot_state {
102 SLOT_CLEAN = 0,
103 SLOT_DIRTY = 1,
104 SLOT_USED = 2,
105};
106
107void __weak *alloc_insn_page(void)
108{
109 return module_alloc(PAGE_SIZE);
110}
111
112void __weak free_insn_page(void *page)
113{
114 module_memfree(page);
115}
116
117struct kprobe_insn_cache kprobe_insn_slots = {
118 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
119 .alloc = alloc_insn_page,
120 .free = free_insn_page,
121 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
122 .insn_size = MAX_INSN_SIZE,
123 .nr_garbage = 0,
124};
125static int collect_garbage_slots(struct kprobe_insn_cache *c);
126
127
128
129
130
131kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c)
132{
133 struct kprobe_insn_page *kip;
134 kprobe_opcode_t *slot = NULL;
135
136
137 mutex_lock(&c->mutex);
138 retry:
139 rcu_read_lock();
140 list_for_each_entry_rcu(kip, &c->pages, list) {
141 if (kip->nused < slots_per_page(c)) {
142 int i;
143 for (i = 0; i < slots_per_page(c); i++) {
144 if (kip->slot_used[i] == SLOT_CLEAN) {
145 kip->slot_used[i] = SLOT_USED;
146 kip->nused++;
147 slot = kip->insns + (i * c->insn_size);
148 rcu_read_unlock();
149 goto out;
150 }
151 }
152
153 kip->nused = slots_per_page(c);
154 WARN_ON(1);
155 }
156 }
157 rcu_read_unlock();
158
159
160 if (c->nr_garbage && collect_garbage_slots(c) == 0)
161 goto retry;
162
163
164 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
165 if (!kip)
166 goto out;
167
168
169
170
171
172
173 kip->insns = c->alloc();
174 if (!kip->insns) {
175 kfree(kip);
176 goto out;
177 }
178 INIT_LIST_HEAD(&kip->list);
179 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
180 kip->slot_used[0] = SLOT_USED;
181 kip->nused = 1;
182 kip->ngarbage = 0;
183 kip->cache = c;
184 list_add_rcu(&kip->list, &c->pages);
185 slot = kip->insns;
186out:
187 mutex_unlock(&c->mutex);
188 return slot;
189}
190
191
192static int collect_one_slot(struct kprobe_insn_page *kip, int idx)
193{
194 kip->slot_used[idx] = SLOT_CLEAN;
195 kip->nused--;
196 if (kip->nused == 0) {
197
198
199
200
201
202
203 if (!list_is_singular(&kip->list)) {
204 list_del_rcu(&kip->list);
205 synchronize_rcu();
206 kip->cache->free(kip->insns);
207 kfree(kip);
208 }
209 return 1;
210 }
211 return 0;
212}
213
214static int collect_garbage_slots(struct kprobe_insn_cache *c)
215{
216 struct kprobe_insn_page *kip, *next;
217
218
219 synchronize_rcu();
220
221 list_for_each_entry_safe(kip, next, &c->pages, list) {
222 int i;
223 if (kip->ngarbage == 0)
224 continue;
225 kip->ngarbage = 0;
226 for (i = 0; i < slots_per_page(c); i++) {
227 if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
228 break;
229 }
230 }
231 c->nr_garbage = 0;
232 return 0;
233}
234
235void __free_insn_slot(struct kprobe_insn_cache *c,
236 kprobe_opcode_t *slot, int dirty)
237{
238 struct kprobe_insn_page *kip;
239 long idx;
240
241 mutex_lock(&c->mutex);
242 rcu_read_lock();
243 list_for_each_entry_rcu(kip, &c->pages, list) {
244 idx = ((long)slot - (long)kip->insns) /
245 (c->insn_size * sizeof(kprobe_opcode_t));
246 if (idx >= 0 && idx < slots_per_page(c))
247 goto out;
248 }
249
250 WARN_ON(1);
251 kip = NULL;
252out:
253 rcu_read_unlock();
254
255 if (kip) {
256
257 WARN_ON(kip->slot_used[idx] != SLOT_USED);
258 if (dirty) {
259 kip->slot_used[idx] = SLOT_DIRTY;
260 kip->ngarbage++;
261 if (++c->nr_garbage > slots_per_page(c))
262 collect_garbage_slots(c);
263 } else {
264 collect_one_slot(kip, idx);
265 }
266 }
267 mutex_unlock(&c->mutex);
268}
269
270
271
272
273
274
275bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
276{
277 struct kprobe_insn_page *kip;
278 bool ret = false;
279
280 rcu_read_lock();
281 list_for_each_entry_rcu(kip, &c->pages, list) {
282 if (addr >= (unsigned long)kip->insns &&
283 addr < (unsigned long)kip->insns + PAGE_SIZE) {
284 ret = true;
285 break;
286 }
287 }
288 rcu_read_unlock();
289
290 return ret;
291}
292
293#ifdef CONFIG_OPTPROBES
294
295struct kprobe_insn_cache kprobe_optinsn_slots = {
296 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
297 .alloc = alloc_insn_page,
298 .free = free_insn_page,
299 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
300
301 .nr_garbage = 0,
302};
303#endif
304#endif
305
306
307static inline void set_kprobe_instance(struct kprobe *kp)
308{
309 __this_cpu_write(kprobe_instance, kp);
310}
311
312static inline void reset_kprobe_instance(void)
313{
314 __this_cpu_write(kprobe_instance, NULL);
315}
316
317
318
319
320
321
322
323struct kprobe *get_kprobe(void *addr)
324{
325 struct hlist_head *head;
326 struct kprobe *p;
327
328 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
329 hlist_for_each_entry_rcu(p, head, hlist) {
330 if (p->addr == addr)
331 return p;
332 }
333
334 return NULL;
335}
336NOKPROBE_SYMBOL(get_kprobe);
337
338static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
339
340
341static inline int kprobe_aggrprobe(struct kprobe *p)
342{
343 return p->pre_handler == aggr_pre_handler;
344}
345
346
347static inline int kprobe_unused(struct kprobe *p)
348{
349 return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
350 list_empty(&p->list);
351}
352
353
354
355
356static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
357{
358 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
359 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
360}
361
362#ifdef CONFIG_OPTPROBES
363
364static bool kprobes_allow_optimization;
365
366
367
368
369
370void opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
371{
372 struct kprobe *kp;
373
374 list_for_each_entry_rcu(kp, &p->list, list) {
375 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
376 set_kprobe_instance(kp);
377 kp->pre_handler(kp, regs);
378 }
379 reset_kprobe_instance();
380 }
381}
382NOKPROBE_SYMBOL(opt_pre_handler);
383
384
385static void free_aggr_kprobe(struct kprobe *p)
386{
387 struct optimized_kprobe *op;
388
389 op = container_of(p, struct optimized_kprobe, kp);
390 arch_remove_optimized_kprobe(op);
391 arch_remove_kprobe(p);
392 kfree(op);
393}
394
395
396static inline int kprobe_optready(struct kprobe *p)
397{
398 struct optimized_kprobe *op;
399
400 if (kprobe_aggrprobe(p)) {
401 op = container_of(p, struct optimized_kprobe, kp);
402 return arch_prepared_optinsn(&op->optinsn);
403 }
404
405 return 0;
406}
407
408
409static inline int kprobe_disarmed(struct kprobe *p)
410{
411 struct optimized_kprobe *op;
412
413
414 if (!kprobe_aggrprobe(p))
415 return kprobe_disabled(p);
416
417 op = container_of(p, struct optimized_kprobe, kp);
418
419 return kprobe_disabled(p) && list_empty(&op->list);
420}
421
422
423static int kprobe_queued(struct kprobe *p)
424{
425 struct optimized_kprobe *op;
426
427 if (kprobe_aggrprobe(p)) {
428 op = container_of(p, struct optimized_kprobe, kp);
429 if (!list_empty(&op->list))
430 return 1;
431 }
432 return 0;
433}
434
435
436
437
438
439static struct kprobe *get_optimized_kprobe(unsigned long addr)
440{
441 int i;
442 struct kprobe *p = NULL;
443 struct optimized_kprobe *op;
444
445
446 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
447 p = get_kprobe((void *)(addr - i));
448
449 if (p && kprobe_optready(p)) {
450 op = container_of(p, struct optimized_kprobe, kp);
451 if (arch_within_optimized_kprobe(op, addr))
452 return p;
453 }
454
455 return NULL;
456}
457
458
459static LIST_HEAD(optimizing_list);
460static LIST_HEAD(unoptimizing_list);
461static LIST_HEAD(freeing_list);
462
463static void kprobe_optimizer(struct work_struct *work);
464static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
465#define OPTIMIZE_DELAY 5
466
467
468
469
470
471static void do_optimize_kprobes(void)
472{
473 lockdep_assert_held(&text_mutex);
474
475
476
477
478
479
480
481
482
483
484 lockdep_assert_cpus_held();
485
486
487 if (kprobes_all_disarmed || !kprobes_allow_optimization ||
488 list_empty(&optimizing_list))
489 return;
490
491 arch_optimize_kprobes(&optimizing_list);
492}
493
494
495
496
497
498static void do_unoptimize_kprobes(void)
499{
500 struct optimized_kprobe *op, *tmp;
501
502 lockdep_assert_held(&text_mutex);
503
504 lockdep_assert_cpus_held();
505
506
507 if (list_empty(&unoptimizing_list))
508 return;
509
510 arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
511
512 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
513
514 if (kprobe_disabled(&op->kp))
515 arch_disarm_kprobe(&op->kp);
516 if (kprobe_unused(&op->kp)) {
517
518
519
520
521
522 hlist_del_rcu(&op->kp.hlist);
523 } else
524 list_del_init(&op->list);
525 }
526}
527
528
529static void do_free_cleaned_kprobes(void)
530{
531 struct optimized_kprobe *op, *tmp;
532
533 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
534 list_del_init(&op->list);
535 if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) {
536
537
538
539
540 continue;
541 }
542 free_aggr_kprobe(&op->kp);
543 }
544}
545
546
547static void kick_kprobe_optimizer(void)
548{
549 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
550}
551
552
553static void kprobe_optimizer(struct work_struct *work)
554{
555 mutex_lock(&kprobe_mutex);
556 cpus_read_lock();
557 mutex_lock(&text_mutex);
558
559 mutex_lock(&module_mutex);
560
561
562
563
564
565 do_unoptimize_kprobes();
566
567
568
569
570
571
572
573
574
575
576 synchronize_rcu_tasks();
577
578
579 do_optimize_kprobes();
580
581
582 do_free_cleaned_kprobes();
583
584 mutex_unlock(&module_mutex);
585 mutex_unlock(&text_mutex);
586 cpus_read_unlock();
587 mutex_unlock(&kprobe_mutex);
588
589
590 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
591 kick_kprobe_optimizer();
592}
593
594
595void wait_for_kprobe_optimizer(void)
596{
597 mutex_lock(&kprobe_mutex);
598
599 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
600 mutex_unlock(&kprobe_mutex);
601
602
603 flush_delayed_work(&optimizing_work);
604
605 cpu_relax();
606
607 mutex_lock(&kprobe_mutex);
608 }
609
610 mutex_unlock(&kprobe_mutex);
611}
612
613
614static void optimize_kprobe(struct kprobe *p)
615{
616 struct optimized_kprobe *op;
617
618
619 if (!kprobe_optready(p) || !kprobes_allow_optimization ||
620 (kprobe_disabled(p) || kprobes_all_disarmed))
621 return;
622
623
624 if (p->post_handler)
625 return;
626
627 op = container_of(p, struct optimized_kprobe, kp);
628
629
630 if (arch_check_optimized_kprobe(op) < 0)
631 return;
632
633
634 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
635 return;
636 op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
637
638 if (!list_empty(&op->list))
639
640 list_del_init(&op->list);
641 else {
642 list_add(&op->list, &optimizing_list);
643 kick_kprobe_optimizer();
644 }
645}
646
647
648static void force_unoptimize_kprobe(struct optimized_kprobe *op)
649{
650 lockdep_assert_cpus_held();
651 arch_unoptimize_kprobe(op);
652 if (kprobe_disabled(&op->kp))
653 arch_disarm_kprobe(&op->kp);
654}
655
656
657static void unoptimize_kprobe(struct kprobe *p, bool force)
658{
659 struct optimized_kprobe *op;
660
661 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
662 return;
663
664 op = container_of(p, struct optimized_kprobe, kp);
665 if (!kprobe_optimized(p)) {
666
667 if (force && !list_empty(&op->list)) {
668
669
670
671
672
673 list_del_init(&op->list);
674 force_unoptimize_kprobe(op);
675 }
676 return;
677 }
678
679 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
680 if (!list_empty(&op->list)) {
681
682 list_del_init(&op->list);
683 return;
684 }
685
686 if (force)
687
688 force_unoptimize_kprobe(op);
689 else {
690 list_add(&op->list, &unoptimizing_list);
691 kick_kprobe_optimizer();
692 }
693}
694
695
696static int reuse_unused_kprobe(struct kprobe *ap)
697{
698 struct optimized_kprobe *op;
699
700
701
702
703
704 op = container_of(ap, struct optimized_kprobe, kp);
705 WARN_ON_ONCE(list_empty(&op->list));
706
707 ap->flags &= ~KPROBE_FLAG_DISABLED;
708
709 if (!kprobe_optready(ap))
710 return -EINVAL;
711
712 optimize_kprobe(ap);
713 return 0;
714}
715
716
717static void kill_optimized_kprobe(struct kprobe *p)
718{
719 struct optimized_kprobe *op;
720
721 op = container_of(p, struct optimized_kprobe, kp);
722 if (!list_empty(&op->list))
723
724 list_del_init(&op->list);
725 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
726
727 if (kprobe_unused(p)) {
728
729 list_add(&op->list, &freeing_list);
730
731
732
733
734
735 hlist_del_rcu(&op->kp.hlist);
736 }
737
738
739 arch_remove_optimized_kprobe(op);
740}
741
742static inline
743void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
744{
745 if (!kprobe_ftrace(p))
746 arch_prepare_optimized_kprobe(op, p);
747}
748
749
750static void prepare_optimized_kprobe(struct kprobe *p)
751{
752 struct optimized_kprobe *op;
753
754 op = container_of(p, struct optimized_kprobe, kp);
755 __prepare_optimized_kprobe(op, p);
756}
757
758
759static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
760{
761 struct optimized_kprobe *op;
762
763 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
764 if (!op)
765 return NULL;
766
767 INIT_LIST_HEAD(&op->list);
768 op->kp.addr = p->addr;
769 __prepare_optimized_kprobe(op, p);
770
771 return &op->kp;
772}
773
774static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
775
776
777
778
779
780static void try_to_optimize_kprobe(struct kprobe *p)
781{
782 struct kprobe *ap;
783 struct optimized_kprobe *op;
784
785
786 if (kprobe_ftrace(p))
787 return;
788
789
790 cpus_read_lock();
791 jump_label_lock();
792 mutex_lock(&text_mutex);
793
794 ap = alloc_aggr_kprobe(p);
795 if (!ap)
796 goto out;
797
798 op = container_of(ap, struct optimized_kprobe, kp);
799 if (!arch_prepared_optinsn(&op->optinsn)) {
800
801 arch_remove_optimized_kprobe(op);
802 kfree(op);
803 goto out;
804 }
805
806 init_aggr_kprobe(ap, p);
807 optimize_kprobe(ap);
808
809out:
810 mutex_unlock(&text_mutex);
811 jump_label_unlock();
812 cpus_read_unlock();
813}
814
815#ifdef CONFIG_SYSCTL
816static void optimize_all_kprobes(void)
817{
818 struct hlist_head *head;
819 struct kprobe *p;
820 unsigned int i;
821
822 mutex_lock(&kprobe_mutex);
823
824 if (kprobes_allow_optimization)
825 goto out;
826
827 cpus_read_lock();
828 kprobes_allow_optimization = true;
829 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
830 head = &kprobe_table[i];
831 hlist_for_each_entry_rcu(p, head, hlist)
832 if (!kprobe_disabled(p))
833 optimize_kprobe(p);
834 }
835 cpus_read_unlock();
836 printk(KERN_INFO "Kprobes globally optimized\n");
837out:
838 mutex_unlock(&kprobe_mutex);
839}
840
841static void unoptimize_all_kprobes(void)
842{
843 struct hlist_head *head;
844 struct kprobe *p;
845 unsigned int i;
846
847 mutex_lock(&kprobe_mutex);
848
849 if (!kprobes_allow_optimization) {
850 mutex_unlock(&kprobe_mutex);
851 return;
852 }
853
854 cpus_read_lock();
855 kprobes_allow_optimization = false;
856 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
857 head = &kprobe_table[i];
858 hlist_for_each_entry_rcu(p, head, hlist) {
859 if (!kprobe_disabled(p))
860 unoptimize_kprobe(p, false);
861 }
862 }
863 cpus_read_unlock();
864 mutex_unlock(&kprobe_mutex);
865
866
867 wait_for_kprobe_optimizer();
868 printk(KERN_INFO "Kprobes globally unoptimized\n");
869}
870
871static DEFINE_MUTEX(kprobe_sysctl_mutex);
872int sysctl_kprobes_optimization;
873int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
874 void __user *buffer, size_t *length,
875 loff_t *ppos)
876{
877 int ret;
878
879 mutex_lock(&kprobe_sysctl_mutex);
880 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
881 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
882
883 if (sysctl_kprobes_optimization)
884 optimize_all_kprobes();
885 else
886 unoptimize_all_kprobes();
887 mutex_unlock(&kprobe_sysctl_mutex);
888
889 return ret;
890}
891#endif
892
893
894static void __arm_kprobe(struct kprobe *p)
895{
896 struct kprobe *_p;
897
898
899 _p = get_optimized_kprobe((unsigned long)p->addr);
900 if (unlikely(_p))
901
902 unoptimize_kprobe(_p, true);
903
904 arch_arm_kprobe(p);
905 optimize_kprobe(p);
906}
907
908
909static void __disarm_kprobe(struct kprobe *p, bool reopt)
910{
911 struct kprobe *_p;
912
913
914 unoptimize_kprobe(p, kprobes_all_disarmed);
915
916 if (!kprobe_queued(p)) {
917 arch_disarm_kprobe(p);
918
919 _p = get_optimized_kprobe((unsigned long)p->addr);
920 if (unlikely(_p) && reopt)
921 optimize_kprobe(_p);
922 }
923
924}
925
926#else
927
928#define optimize_kprobe(p) do {} while (0)
929#define unoptimize_kprobe(p, f) do {} while (0)
930#define kill_optimized_kprobe(p) do {} while (0)
931#define prepare_optimized_kprobe(p) do {} while (0)
932#define try_to_optimize_kprobe(p) do {} while (0)
933#define __arm_kprobe(p) arch_arm_kprobe(p)
934#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
935#define kprobe_disarmed(p) kprobe_disabled(p)
936#define wait_for_kprobe_optimizer() do {} while (0)
937
938static int reuse_unused_kprobe(struct kprobe *ap)
939{
940
941
942
943
944
945
946 printk(KERN_ERR "Error: There should be no unused kprobe here.\n");
947 return -EINVAL;
948}
949
950static void free_aggr_kprobe(struct kprobe *p)
951{
952 arch_remove_kprobe(p);
953 kfree(p);
954}
955
956static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
957{
958 return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
959}
960#endif
961
962#ifdef CONFIG_KPROBES_ON_FTRACE
963static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
964 .func = kprobe_ftrace_handler,
965 .flags = FTRACE_OPS_FL_SAVE_REGS,
966};
967
968static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = {
969 .func = kprobe_ftrace_handler,
970 .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
971};
972
973static int kprobe_ipmodify_enabled;
974static int kprobe_ftrace_enabled;
975
976
977static int prepare_kprobe(struct kprobe *p)
978{
979 if (!kprobe_ftrace(p))
980 return arch_prepare_kprobe(p);
981
982 return arch_prepare_kprobe_ftrace(p);
983}
984
985
986static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
987 int *cnt)
988{
989 int ret = 0;
990
991 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0);
992 if (ret) {
993 pr_debug("Failed to arm kprobe-ftrace at %pS (%d)\n",
994 p->addr, ret);
995 return ret;
996 }
997
998 if (*cnt == 0) {
999 ret = register_ftrace_function(ops);
1000 if (ret) {
1001 pr_debug("Failed to init kprobe-ftrace (%d)\n", ret);
1002 goto err_ftrace;
1003 }
1004 }
1005
1006 (*cnt)++;
1007 return ret;
1008
1009err_ftrace:
1010
1011
1012
1013
1014 ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1015 return ret;
1016}
1017
1018static int arm_kprobe_ftrace(struct kprobe *p)
1019{
1020 bool ipmodify = (p->post_handler != NULL);
1021
1022 return __arm_kprobe_ftrace(p,
1023 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1024 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1025}
1026
1027
1028static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1029 int *cnt)
1030{
1031 int ret = 0;
1032
1033 if (*cnt == 1) {
1034 ret = unregister_ftrace_function(ops);
1035 if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (%d)\n", ret))
1036 return ret;
1037 }
1038
1039 (*cnt)--;
1040
1041 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1042 WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (%d)\n",
1043 p->addr, ret);
1044 return ret;
1045}
1046
1047static int disarm_kprobe_ftrace(struct kprobe *p)
1048{
1049 bool ipmodify = (p->post_handler != NULL);
1050
1051 return __disarm_kprobe_ftrace(p,
1052 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1053 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1054}
1055#else
1056#define prepare_kprobe(p) arch_prepare_kprobe(p)
1057#define arm_kprobe_ftrace(p) (-ENODEV)
1058#define disarm_kprobe_ftrace(p) (-ENODEV)
1059#endif
1060
1061
1062static int arm_kprobe(struct kprobe *kp)
1063{
1064 if (unlikely(kprobe_ftrace(kp)))
1065 return arm_kprobe_ftrace(kp);
1066
1067 cpus_read_lock();
1068 mutex_lock(&text_mutex);
1069 __arm_kprobe(kp);
1070 mutex_unlock(&text_mutex);
1071 cpus_read_unlock();
1072
1073 return 0;
1074}
1075
1076
1077static int disarm_kprobe(struct kprobe *kp, bool reopt)
1078{
1079 if (unlikely(kprobe_ftrace(kp)))
1080 return disarm_kprobe_ftrace(kp);
1081
1082 cpus_read_lock();
1083 mutex_lock(&text_mutex);
1084 __disarm_kprobe(kp, reopt);
1085 mutex_unlock(&text_mutex);
1086 cpus_read_unlock();
1087
1088 return 0;
1089}
1090
1091
1092
1093
1094
1095static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
1096{
1097 struct kprobe *kp;
1098
1099 list_for_each_entry_rcu(kp, &p->list, list) {
1100 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
1101 set_kprobe_instance(kp);
1102 if (kp->pre_handler(kp, regs))
1103 return 1;
1104 }
1105 reset_kprobe_instance();
1106 }
1107 return 0;
1108}
1109NOKPROBE_SYMBOL(aggr_pre_handler);
1110
1111static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1112 unsigned long flags)
1113{
1114 struct kprobe *kp;
1115
1116 list_for_each_entry_rcu(kp, &p->list, list) {
1117 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
1118 set_kprobe_instance(kp);
1119 kp->post_handler(kp, regs, flags);
1120 reset_kprobe_instance();
1121 }
1122 }
1123}
1124NOKPROBE_SYMBOL(aggr_post_handler);
1125
1126static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
1127 int trapnr)
1128{
1129 struct kprobe *cur = __this_cpu_read(kprobe_instance);
1130
1131
1132
1133
1134
1135 if (cur && cur->fault_handler) {
1136 if (cur->fault_handler(cur, regs, trapnr))
1137 return 1;
1138 }
1139 return 0;
1140}
1141NOKPROBE_SYMBOL(aggr_fault_handler);
1142
1143
1144void kprobes_inc_nmissed_count(struct kprobe *p)
1145{
1146 struct kprobe *kp;
1147 if (!kprobe_aggrprobe(p)) {
1148 p->nmissed++;
1149 } else {
1150 list_for_each_entry_rcu(kp, &p->list, list)
1151 kp->nmissed++;
1152 }
1153 return;
1154}
1155NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1156
1157void recycle_rp_inst(struct kretprobe_instance *ri,
1158 struct hlist_head *head)
1159{
1160 struct kretprobe *rp = ri->rp;
1161
1162
1163 hlist_del(&ri->hlist);
1164 INIT_HLIST_NODE(&ri->hlist);
1165 if (likely(rp)) {
1166 raw_spin_lock(&rp->lock);
1167 hlist_add_head(&ri->hlist, &rp->free_instances);
1168 raw_spin_unlock(&rp->lock);
1169 } else
1170
1171 hlist_add_head(&ri->hlist, head);
1172}
1173NOKPROBE_SYMBOL(recycle_rp_inst);
1174
1175void kretprobe_hash_lock(struct task_struct *tsk,
1176 struct hlist_head **head, unsigned long *flags)
1177__acquires(hlist_lock)
1178{
1179 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
1180 raw_spinlock_t *hlist_lock;
1181
1182 *head = &kretprobe_inst_table[hash];
1183 hlist_lock = kretprobe_table_lock_ptr(hash);
1184 raw_spin_lock_irqsave(hlist_lock, *flags);
1185}
1186NOKPROBE_SYMBOL(kretprobe_hash_lock);
1187
1188static void kretprobe_table_lock(unsigned long hash,
1189 unsigned long *flags)
1190__acquires(hlist_lock)
1191{
1192 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1193 raw_spin_lock_irqsave(hlist_lock, *flags);
1194}
1195NOKPROBE_SYMBOL(kretprobe_table_lock);
1196
1197void kretprobe_hash_unlock(struct task_struct *tsk,
1198 unsigned long *flags)
1199__releases(hlist_lock)
1200{
1201 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
1202 raw_spinlock_t *hlist_lock;
1203
1204 hlist_lock = kretprobe_table_lock_ptr(hash);
1205 raw_spin_unlock_irqrestore(hlist_lock, *flags);
1206}
1207NOKPROBE_SYMBOL(kretprobe_hash_unlock);
1208
1209static void kretprobe_table_unlock(unsigned long hash,
1210 unsigned long *flags)
1211__releases(hlist_lock)
1212{
1213 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1214 raw_spin_unlock_irqrestore(hlist_lock, *flags);
1215}
1216NOKPROBE_SYMBOL(kretprobe_table_unlock);
1217
1218
1219
1220
1221
1222
1223
1224void kprobe_flush_task(struct task_struct *tk)
1225{
1226 struct kretprobe_instance *ri;
1227 struct hlist_head *head, empty_rp;
1228 struct hlist_node *tmp;
1229 unsigned long hash, flags = 0;
1230
1231 if (unlikely(!kprobes_initialized))
1232
1233 return;
1234
1235 INIT_HLIST_HEAD(&empty_rp);
1236 hash = hash_ptr(tk, KPROBE_HASH_BITS);
1237 head = &kretprobe_inst_table[hash];
1238 kretprobe_table_lock(hash, &flags);
1239 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
1240 if (ri->task == tk)
1241 recycle_rp_inst(ri, &empty_rp);
1242 }
1243 kretprobe_table_unlock(hash, &flags);
1244 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
1245 hlist_del(&ri->hlist);
1246 kfree(ri);
1247 }
1248}
1249NOKPROBE_SYMBOL(kprobe_flush_task);
1250
1251static inline void free_rp_inst(struct kretprobe *rp)
1252{
1253 struct kretprobe_instance *ri;
1254 struct hlist_node *next;
1255
1256 hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) {
1257 hlist_del(&ri->hlist);
1258 kfree(ri);
1259 }
1260}
1261
1262static void cleanup_rp_inst(struct kretprobe *rp)
1263{
1264 unsigned long flags, hash;
1265 struct kretprobe_instance *ri;
1266 struct hlist_node *next;
1267 struct hlist_head *head;
1268
1269
1270 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
1271 kretprobe_table_lock(hash, &flags);
1272 head = &kretprobe_inst_table[hash];
1273 hlist_for_each_entry_safe(ri, next, head, hlist) {
1274 if (ri->rp == rp)
1275 ri->rp = NULL;
1276 }
1277 kretprobe_table_unlock(hash, &flags);
1278 }
1279 free_rp_inst(rp);
1280}
1281NOKPROBE_SYMBOL(cleanup_rp_inst);
1282
1283
1284static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)
1285{
1286 if (p->post_handler)
1287 unoptimize_kprobe(ap, true);
1288
1289 list_add_rcu(&p->list, &ap->list);
1290 if (p->post_handler && !ap->post_handler)
1291 ap->post_handler = aggr_post_handler;
1292
1293 return 0;
1294}
1295
1296
1297
1298
1299
1300static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
1301{
1302
1303 copy_kprobe(p, ap);
1304 flush_insn_slot(ap);
1305 ap->addr = p->addr;
1306 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1307 ap->pre_handler = aggr_pre_handler;
1308 ap->fault_handler = aggr_fault_handler;
1309
1310 if (p->post_handler && !kprobe_gone(p))
1311 ap->post_handler = aggr_post_handler;
1312
1313 INIT_LIST_HEAD(&ap->list);
1314 INIT_HLIST_NODE(&ap->hlist);
1315
1316 list_add_rcu(&p->list, &ap->list);
1317 hlist_replace_rcu(&p->hlist, &ap->hlist);
1318}
1319
1320
1321
1322
1323
1324static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p)
1325{
1326 int ret = 0;
1327 struct kprobe *ap = orig_p;
1328
1329 cpus_read_lock();
1330
1331
1332 jump_label_lock();
1333 mutex_lock(&text_mutex);
1334
1335 if (!kprobe_aggrprobe(orig_p)) {
1336
1337 ap = alloc_aggr_kprobe(orig_p);
1338 if (!ap) {
1339 ret = -ENOMEM;
1340 goto out;
1341 }
1342 init_aggr_kprobe(ap, orig_p);
1343 } else if (kprobe_unused(ap)) {
1344
1345 ret = reuse_unused_kprobe(ap);
1346 if (ret)
1347 goto out;
1348 }
1349
1350 if (kprobe_gone(ap)) {
1351
1352
1353
1354
1355
1356
1357 ret = arch_prepare_kprobe(ap);
1358 if (ret)
1359
1360
1361
1362
1363
1364 goto out;
1365
1366
1367 prepare_optimized_kprobe(ap);
1368
1369
1370
1371
1372
1373 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1374 | KPROBE_FLAG_DISABLED;
1375 }
1376
1377
1378 copy_kprobe(ap, p);
1379 ret = add_new_kprobe(ap, p);
1380
1381out:
1382 mutex_unlock(&text_mutex);
1383 jump_label_unlock();
1384 cpus_read_unlock();
1385
1386 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1387 ap->flags &= ~KPROBE_FLAG_DISABLED;
1388 if (!kprobes_all_disarmed) {
1389
1390 ret = arm_kprobe(ap);
1391 if (ret) {
1392 ap->flags |= KPROBE_FLAG_DISABLED;
1393 list_del_rcu(&p->list);
1394 synchronize_rcu();
1395 }
1396 }
1397 }
1398 return ret;
1399}
1400
1401bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1402{
1403
1404 return addr >= (unsigned long)__kprobes_text_start &&
1405 addr < (unsigned long)__kprobes_text_end;
1406}
1407
1408static bool __within_kprobe_blacklist(unsigned long addr)
1409{
1410 struct kprobe_blacklist_entry *ent;
1411
1412 if (arch_within_kprobe_blacklist(addr))
1413 return true;
1414
1415
1416
1417
1418 list_for_each_entry(ent, &kprobe_blacklist, list) {
1419 if (addr >= ent->start_addr && addr < ent->end_addr)
1420 return true;
1421 }
1422 return false;
1423}
1424
1425bool within_kprobe_blacklist(unsigned long addr)
1426{
1427 char symname[KSYM_NAME_LEN], *p;
1428
1429 if (__within_kprobe_blacklist(addr))
1430 return true;
1431
1432
1433 if (!lookup_symbol_name(addr, symname)) {
1434 p = strchr(symname, '.');
1435 if (!p)
1436 return false;
1437 *p = '\0';
1438 addr = (unsigned long)kprobe_lookup_name(symname, 0);
1439 if (addr)
1440 return __within_kprobe_blacklist(addr);
1441 }
1442 return false;
1443}
1444
1445
1446
1447
1448
1449
1450
1451static kprobe_opcode_t *_kprobe_addr(kprobe_opcode_t *addr,
1452 const char *symbol_name, unsigned int offset)
1453{
1454 if ((symbol_name && addr) || (!symbol_name && !addr))
1455 goto invalid;
1456
1457 if (symbol_name) {
1458 addr = kprobe_lookup_name(symbol_name, offset);
1459 if (!addr)
1460 return ERR_PTR(-ENOENT);
1461 }
1462
1463 addr = (kprobe_opcode_t *)(((char *)addr) + offset);
1464 if (addr)
1465 return addr;
1466
1467invalid:
1468 return ERR_PTR(-EINVAL);
1469}
1470
1471static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
1472{
1473 return _kprobe_addr(p->addr, p->symbol_name, p->offset);
1474}
1475
1476
1477static struct kprobe *__get_valid_kprobe(struct kprobe *p)
1478{
1479 struct kprobe *ap, *list_p;
1480
1481 ap = get_kprobe(p->addr);
1482 if (unlikely(!ap))
1483 return NULL;
1484
1485 if (p != ap) {
1486 list_for_each_entry_rcu(list_p, &ap->list, list)
1487 if (list_p == p)
1488
1489 goto valid;
1490 return NULL;
1491 }
1492valid:
1493 return ap;
1494}
1495
1496
1497static inline int check_kprobe_rereg(struct kprobe *p)
1498{
1499 int ret = 0;
1500
1501 mutex_lock(&kprobe_mutex);
1502 if (__get_valid_kprobe(p))
1503 ret = -EINVAL;
1504 mutex_unlock(&kprobe_mutex);
1505
1506 return ret;
1507}
1508
1509int __weak arch_check_ftrace_location(struct kprobe *p)
1510{
1511 unsigned long ftrace_addr;
1512
1513 ftrace_addr = ftrace_location((unsigned long)p->addr);
1514 if (ftrace_addr) {
1515#ifdef CONFIG_KPROBES_ON_FTRACE
1516
1517 if ((unsigned long)p->addr != ftrace_addr)
1518 return -EILSEQ;
1519 p->flags |= KPROBE_FLAG_FTRACE;
1520#else
1521 return -EINVAL;
1522#endif
1523 }
1524 return 0;
1525}
1526
1527static int check_kprobe_address_safe(struct kprobe *p,
1528 struct module **probed_mod)
1529{
1530 int ret;
1531
1532 ret = arch_check_ftrace_location(p);
1533 if (ret)
1534 return ret;
1535 jump_label_lock();
1536 preempt_disable();
1537
1538
1539 if (!kernel_text_address((unsigned long) p->addr) ||
1540 within_kprobe_blacklist((unsigned long) p->addr) ||
1541 jump_label_text_reserved(p->addr, p->addr) ||
1542 find_bug((unsigned long)p->addr)) {
1543 ret = -EINVAL;
1544 goto out;
1545 }
1546
1547
1548 *probed_mod = __module_text_address((unsigned long) p->addr);
1549 if (*probed_mod) {
1550
1551
1552
1553
1554 if (unlikely(!try_module_get(*probed_mod))) {
1555 ret = -ENOENT;
1556 goto out;
1557 }
1558
1559
1560
1561
1562
1563 if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1564 (*probed_mod)->state != MODULE_STATE_COMING) {
1565 module_put(*probed_mod);
1566 *probed_mod = NULL;
1567 ret = -ENOENT;
1568 }
1569 }
1570out:
1571 preempt_enable();
1572 jump_label_unlock();
1573
1574 return ret;
1575}
1576
1577int register_kprobe(struct kprobe *p)
1578{
1579 int ret;
1580 struct kprobe *old_p;
1581 struct module *probed_mod;
1582 kprobe_opcode_t *addr;
1583
1584
1585 addr = kprobe_addr(p);
1586 if (IS_ERR(addr))
1587 return PTR_ERR(addr);
1588 p->addr = addr;
1589
1590 ret = check_kprobe_rereg(p);
1591 if (ret)
1592 return ret;
1593
1594
1595 p->flags &= KPROBE_FLAG_DISABLED;
1596 p->nmissed = 0;
1597 INIT_LIST_HEAD(&p->list);
1598
1599 ret = check_kprobe_address_safe(p, &probed_mod);
1600 if (ret)
1601 return ret;
1602
1603 mutex_lock(&kprobe_mutex);
1604
1605 old_p = get_kprobe(p->addr);
1606 if (old_p) {
1607
1608 ret = register_aggr_kprobe(old_p, p);
1609 goto out;
1610 }
1611
1612 cpus_read_lock();
1613
1614 mutex_lock(&text_mutex);
1615 ret = prepare_kprobe(p);
1616 mutex_unlock(&text_mutex);
1617 cpus_read_unlock();
1618 if (ret)
1619 goto out;
1620
1621 INIT_HLIST_NODE(&p->hlist);
1622 hlist_add_head_rcu(&p->hlist,
1623 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1624
1625 if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
1626 ret = arm_kprobe(p);
1627 if (ret) {
1628 hlist_del_rcu(&p->hlist);
1629 synchronize_rcu();
1630 goto out;
1631 }
1632 }
1633
1634
1635 try_to_optimize_kprobe(p);
1636out:
1637 mutex_unlock(&kprobe_mutex);
1638
1639 if (probed_mod)
1640 module_put(probed_mod);
1641
1642 return ret;
1643}
1644EXPORT_SYMBOL_GPL(register_kprobe);
1645
1646
1647static int aggr_kprobe_disabled(struct kprobe *ap)
1648{
1649 struct kprobe *kp;
1650
1651 list_for_each_entry_rcu(kp, &ap->list, list)
1652 if (!kprobe_disabled(kp))
1653
1654
1655
1656
1657 return 0;
1658
1659 return 1;
1660}
1661
1662
1663static struct kprobe *__disable_kprobe(struct kprobe *p)
1664{
1665 struct kprobe *orig_p;
1666 int ret;
1667
1668
1669 orig_p = __get_valid_kprobe(p);
1670 if (unlikely(orig_p == NULL))
1671 return ERR_PTR(-EINVAL);
1672
1673 if (!kprobe_disabled(p)) {
1674
1675 if (p != orig_p)
1676 p->flags |= KPROBE_FLAG_DISABLED;
1677
1678
1679 if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
1680
1681
1682
1683
1684
1685 if (!kprobes_all_disarmed) {
1686 ret = disarm_kprobe(orig_p, true);
1687 if (ret) {
1688 p->flags &= ~KPROBE_FLAG_DISABLED;
1689 return ERR_PTR(ret);
1690 }
1691 }
1692 orig_p->flags |= KPROBE_FLAG_DISABLED;
1693 }
1694 }
1695
1696 return orig_p;
1697}
1698
1699
1700
1701
1702static int __unregister_kprobe_top(struct kprobe *p)
1703{
1704 struct kprobe *ap, *list_p;
1705
1706
1707 ap = __disable_kprobe(p);
1708 if (IS_ERR(ap))
1709 return PTR_ERR(ap);
1710
1711 if (ap == p)
1712
1713
1714
1715
1716 goto disarmed;
1717
1718
1719 WARN_ON(!kprobe_aggrprobe(ap));
1720
1721 if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1722
1723
1724
1725
1726 goto disarmed;
1727 else {
1728
1729 if (p->post_handler && !kprobe_gone(p)) {
1730 list_for_each_entry_rcu(list_p, &ap->list, list) {
1731 if ((list_p != p) && (list_p->post_handler))
1732 goto noclean;
1733 }
1734 ap->post_handler = NULL;
1735 }
1736noclean:
1737
1738
1739
1740
1741 list_del_rcu(&p->list);
1742 if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1743
1744
1745
1746
1747 optimize_kprobe(ap);
1748 }
1749 return 0;
1750
1751disarmed:
1752 hlist_del_rcu(&ap->hlist);
1753 return 0;
1754}
1755
1756static void __unregister_kprobe_bottom(struct kprobe *p)
1757{
1758 struct kprobe *ap;
1759
1760 if (list_empty(&p->list))
1761
1762 arch_remove_kprobe(p);
1763 else if (list_is_singular(&p->list)) {
1764
1765 ap = list_entry(p->list.next, struct kprobe, list);
1766 list_del(&p->list);
1767 free_aggr_kprobe(ap);
1768 }
1769
1770}
1771
1772int register_kprobes(struct kprobe **kps, int num)
1773{
1774 int i, ret = 0;
1775
1776 if (num <= 0)
1777 return -EINVAL;
1778 for (i = 0; i < num; i++) {
1779 ret = register_kprobe(kps[i]);
1780 if (ret < 0) {
1781 if (i > 0)
1782 unregister_kprobes(kps, i);
1783 break;
1784 }
1785 }
1786 return ret;
1787}
1788EXPORT_SYMBOL_GPL(register_kprobes);
1789
1790void unregister_kprobe(struct kprobe *p)
1791{
1792 unregister_kprobes(&p, 1);
1793}
1794EXPORT_SYMBOL_GPL(unregister_kprobe);
1795
1796void unregister_kprobes(struct kprobe **kps, int num)
1797{
1798 int i;
1799
1800 if (num <= 0)
1801 return;
1802 mutex_lock(&kprobe_mutex);
1803 for (i = 0; i < num; i++)
1804 if (__unregister_kprobe_top(kps[i]) < 0)
1805 kps[i]->addr = NULL;
1806 mutex_unlock(&kprobe_mutex);
1807
1808 synchronize_rcu();
1809 for (i = 0; i < num; i++)
1810 if (kps[i]->addr)
1811 __unregister_kprobe_bottom(kps[i]);
1812}
1813EXPORT_SYMBOL_GPL(unregister_kprobes);
1814
1815int __weak kprobe_exceptions_notify(struct notifier_block *self,
1816 unsigned long val, void *data)
1817{
1818 return NOTIFY_DONE;
1819}
1820NOKPROBE_SYMBOL(kprobe_exceptions_notify);
1821
1822static struct notifier_block kprobe_exceptions_nb = {
1823 .notifier_call = kprobe_exceptions_notify,
1824 .priority = 0x7fffffff
1825};
1826
1827unsigned long __weak arch_deref_entry_point(void *entry)
1828{
1829 return (unsigned long)entry;
1830}
1831
1832#ifdef CONFIG_KRETPROBES
1833
1834
1835
1836
1837static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
1838{
1839 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1840 unsigned long hash, flags = 0;
1841 struct kretprobe_instance *ri;
1842
1843
1844
1845
1846
1847
1848
1849 if (unlikely(in_nmi())) {
1850 rp->nmissed++;
1851 return 0;
1852 }
1853
1854
1855 hash = hash_ptr(current, KPROBE_HASH_BITS);
1856 raw_spin_lock_irqsave(&rp->lock, flags);
1857 if (!hlist_empty(&rp->free_instances)) {
1858 ri = hlist_entry(rp->free_instances.first,
1859 struct kretprobe_instance, hlist);
1860 hlist_del(&ri->hlist);
1861 raw_spin_unlock_irqrestore(&rp->lock, flags);
1862
1863 ri->rp = rp;
1864 ri->task = current;
1865
1866 if (rp->entry_handler && rp->entry_handler(ri, regs)) {
1867 raw_spin_lock_irqsave(&rp->lock, flags);
1868 hlist_add_head(&ri->hlist, &rp->free_instances);
1869 raw_spin_unlock_irqrestore(&rp->lock, flags);
1870 return 0;
1871 }
1872
1873 arch_prepare_kretprobe(ri, regs);
1874
1875
1876 INIT_HLIST_NODE(&ri->hlist);
1877 kretprobe_table_lock(hash, &flags);
1878 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1879 kretprobe_table_unlock(hash, &flags);
1880 } else {
1881 rp->nmissed++;
1882 raw_spin_unlock_irqrestore(&rp->lock, flags);
1883 }
1884 return 0;
1885}
1886NOKPROBE_SYMBOL(pre_handler_kretprobe);
1887
1888bool __weak arch_kprobe_on_func_entry(unsigned long offset)
1889{
1890 return !offset;
1891}
1892
1893bool kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
1894{
1895 kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
1896
1897 if (IS_ERR(kp_addr))
1898 return false;
1899
1900 if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset) ||
1901 !arch_kprobe_on_func_entry(offset))
1902 return false;
1903
1904 return true;
1905}
1906
1907int register_kretprobe(struct kretprobe *rp)
1908{
1909 int ret = 0;
1910 struct kretprobe_instance *inst;
1911 int i;
1912 void *addr;
1913
1914 if (!kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset))
1915 return -EINVAL;
1916
1917 if (kretprobe_blacklist_size) {
1918 addr = kprobe_addr(&rp->kp);
1919 if (IS_ERR(addr))
1920 return PTR_ERR(addr);
1921
1922 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1923 if (kretprobe_blacklist[i].addr == addr)
1924 return -EINVAL;
1925 }
1926 }
1927
1928 rp->kp.pre_handler = pre_handler_kretprobe;
1929 rp->kp.post_handler = NULL;
1930 rp->kp.fault_handler = NULL;
1931
1932
1933 if (rp->maxactive <= 0) {
1934#ifdef CONFIG_PREEMPTION
1935 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
1936#else
1937 rp->maxactive = num_possible_cpus();
1938#endif
1939 }
1940 raw_spin_lock_init(&rp->lock);
1941 INIT_HLIST_HEAD(&rp->free_instances);
1942 for (i = 0; i < rp->maxactive; i++) {
1943 inst = kmalloc(sizeof(struct kretprobe_instance) +
1944 rp->data_size, GFP_KERNEL);
1945 if (inst == NULL) {
1946 free_rp_inst(rp);
1947 return -ENOMEM;
1948 }
1949 INIT_HLIST_NODE(&inst->hlist);
1950 hlist_add_head(&inst->hlist, &rp->free_instances);
1951 }
1952
1953 rp->nmissed = 0;
1954
1955 ret = register_kprobe(&rp->kp);
1956 if (ret != 0)
1957 free_rp_inst(rp);
1958 return ret;
1959}
1960EXPORT_SYMBOL_GPL(register_kretprobe);
1961
1962int register_kretprobes(struct kretprobe **rps, int num)
1963{
1964 int ret = 0, i;
1965
1966 if (num <= 0)
1967 return -EINVAL;
1968 for (i = 0; i < num; i++) {
1969 ret = register_kretprobe(rps[i]);
1970 if (ret < 0) {
1971 if (i > 0)
1972 unregister_kretprobes(rps, i);
1973 break;
1974 }
1975 }
1976 return ret;
1977}
1978EXPORT_SYMBOL_GPL(register_kretprobes);
1979
1980void unregister_kretprobe(struct kretprobe *rp)
1981{
1982 unregister_kretprobes(&rp, 1);
1983}
1984EXPORT_SYMBOL_GPL(unregister_kretprobe);
1985
1986void unregister_kretprobes(struct kretprobe **rps, int num)
1987{
1988 int i;
1989
1990 if (num <= 0)
1991 return;
1992 mutex_lock(&kprobe_mutex);
1993 for (i = 0; i < num; i++)
1994 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1995 rps[i]->kp.addr = NULL;
1996 mutex_unlock(&kprobe_mutex);
1997
1998 synchronize_rcu();
1999 for (i = 0; i < num; i++) {
2000 if (rps[i]->kp.addr) {
2001 __unregister_kprobe_bottom(&rps[i]->kp);
2002 cleanup_rp_inst(rps[i]);
2003 }
2004 }
2005}
2006EXPORT_SYMBOL_GPL(unregister_kretprobes);
2007
2008#else
2009int register_kretprobe(struct kretprobe *rp)
2010{
2011 return -ENOSYS;
2012}
2013EXPORT_SYMBOL_GPL(register_kretprobe);
2014
2015int register_kretprobes(struct kretprobe **rps, int num)
2016{
2017 return -ENOSYS;
2018}
2019EXPORT_SYMBOL_GPL(register_kretprobes);
2020
2021void unregister_kretprobe(struct kretprobe *rp)
2022{
2023}
2024EXPORT_SYMBOL_GPL(unregister_kretprobe);
2025
2026void unregister_kretprobes(struct kretprobe **rps, int num)
2027{
2028}
2029EXPORT_SYMBOL_GPL(unregister_kretprobes);
2030
2031static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2032{
2033 return 0;
2034}
2035NOKPROBE_SYMBOL(pre_handler_kretprobe);
2036
2037#endif
2038
2039
2040static void kill_kprobe(struct kprobe *p)
2041{
2042 struct kprobe *kp;
2043
2044 p->flags |= KPROBE_FLAG_GONE;
2045 if (kprobe_aggrprobe(p)) {
2046
2047
2048
2049
2050 list_for_each_entry_rcu(kp, &p->list, list)
2051 kp->flags |= KPROBE_FLAG_GONE;
2052 p->post_handler = NULL;
2053 kill_optimized_kprobe(p);
2054 }
2055
2056
2057
2058
2059 arch_remove_kprobe(p);
2060}
2061
2062
2063int disable_kprobe(struct kprobe *kp)
2064{
2065 int ret = 0;
2066 struct kprobe *p;
2067
2068 mutex_lock(&kprobe_mutex);
2069
2070
2071 p = __disable_kprobe(kp);
2072 if (IS_ERR(p))
2073 ret = PTR_ERR(p);
2074
2075 mutex_unlock(&kprobe_mutex);
2076 return ret;
2077}
2078EXPORT_SYMBOL_GPL(disable_kprobe);
2079
2080
2081int enable_kprobe(struct kprobe *kp)
2082{
2083 int ret = 0;
2084 struct kprobe *p;
2085
2086 mutex_lock(&kprobe_mutex);
2087
2088
2089 p = __get_valid_kprobe(kp);
2090 if (unlikely(p == NULL)) {
2091 ret = -EINVAL;
2092 goto out;
2093 }
2094
2095 if (kprobe_gone(kp)) {
2096
2097 ret = -EINVAL;
2098 goto out;
2099 }
2100
2101 if (p != kp)
2102 kp->flags &= ~KPROBE_FLAG_DISABLED;
2103
2104 if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2105 p->flags &= ~KPROBE_FLAG_DISABLED;
2106 ret = arm_kprobe(p);
2107 if (ret)
2108 p->flags |= KPROBE_FLAG_DISABLED;
2109 }
2110out:
2111 mutex_unlock(&kprobe_mutex);
2112 return ret;
2113}
2114EXPORT_SYMBOL_GPL(enable_kprobe);
2115
2116
2117void dump_kprobe(struct kprobe *kp)
2118{
2119 pr_err("Dumping kprobe:\n");
2120 pr_err("Name: %s\nOffset: %x\nAddress: %pS\n",
2121 kp->symbol_name, kp->offset, kp->addr);
2122}
2123NOKPROBE_SYMBOL(dump_kprobe);
2124
2125int kprobe_add_ksym_blacklist(unsigned long entry)
2126{
2127 struct kprobe_blacklist_entry *ent;
2128 unsigned long offset = 0, size = 0;
2129
2130 if (!kernel_text_address(entry) ||
2131 !kallsyms_lookup_size_offset(entry, &size, &offset))
2132 return -EINVAL;
2133
2134 ent = kmalloc(sizeof(*ent), GFP_KERNEL);
2135 if (!ent)
2136 return -ENOMEM;
2137 ent->start_addr = entry;
2138 ent->end_addr = entry + size;
2139 INIT_LIST_HEAD(&ent->list);
2140 list_add_tail(&ent->list, &kprobe_blacklist);
2141
2142 return (int)size;
2143}
2144
2145
2146int kprobe_add_area_blacklist(unsigned long start, unsigned long end)
2147{
2148 unsigned long entry;
2149 int ret = 0;
2150
2151 for (entry = start; entry < end; entry += ret) {
2152 ret = kprobe_add_ksym_blacklist(entry);
2153 if (ret < 0)
2154 return ret;
2155 if (ret == 0)
2156 ret = 1;
2157 }
2158 return 0;
2159}
2160
2161int __init __weak arch_populate_kprobe_blacklist(void)
2162{
2163 return 0;
2164}
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174static int __init populate_kprobe_blacklist(unsigned long *start,
2175 unsigned long *end)
2176{
2177 unsigned long entry;
2178 unsigned long *iter;
2179 int ret;
2180
2181 for (iter = start; iter < end; iter++) {
2182 entry = arch_deref_entry_point((void *)*iter);
2183 ret = kprobe_add_ksym_blacklist(entry);
2184 if (ret == -EINVAL)
2185 continue;
2186 if (ret < 0)
2187 return ret;
2188 }
2189
2190
2191 ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start,
2192 (unsigned long)__kprobes_text_end);
2193
2194 return ret ? : arch_populate_kprobe_blacklist();
2195}
2196
2197
2198static int kprobes_module_callback(struct notifier_block *nb,
2199 unsigned long val, void *data)
2200{
2201 struct module *mod = data;
2202 struct hlist_head *head;
2203 struct kprobe *p;
2204 unsigned int i;
2205 int checkcore = (val == MODULE_STATE_GOING);
2206
2207 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
2208 return NOTIFY_DONE;
2209
2210
2211
2212
2213
2214
2215
2216 mutex_lock(&kprobe_mutex);
2217 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2218 head = &kprobe_table[i];
2219 hlist_for_each_entry_rcu(p, head, hlist)
2220 if (within_module_init((unsigned long)p->addr, mod) ||
2221 (checkcore &&
2222 within_module_core((unsigned long)p->addr, mod))) {
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234 kill_kprobe(p);
2235 }
2236 }
2237 mutex_unlock(&kprobe_mutex);
2238 return NOTIFY_DONE;
2239}
2240
2241static struct notifier_block kprobe_module_nb = {
2242 .notifier_call = kprobes_module_callback,
2243 .priority = 0
2244};
2245
2246
2247extern unsigned long __start_kprobe_blacklist[];
2248extern unsigned long __stop_kprobe_blacklist[];
2249
2250static int __init init_kprobes(void)
2251{
2252 int i, err = 0;
2253
2254
2255
2256 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2257 INIT_HLIST_HEAD(&kprobe_table[i]);
2258 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
2259 raw_spin_lock_init(&(kretprobe_table_locks[i].lock));
2260 }
2261
2262 err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2263 __stop_kprobe_blacklist);
2264 if (err) {
2265 pr_err("kprobes: failed to populate blacklist: %d\n", err);
2266 pr_err("Please take care of using kprobes.\n");
2267 }
2268
2269 if (kretprobe_blacklist_size) {
2270
2271 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2272 kretprobe_blacklist[i].addr =
2273 kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
2274 if (!kretprobe_blacklist[i].addr)
2275 printk("kretprobe: lookup failed: %s\n",
2276 kretprobe_blacklist[i].name);
2277 }
2278 }
2279
2280#if defined(CONFIG_OPTPROBES)
2281#if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2282
2283 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2284#endif
2285
2286 kprobes_allow_optimization = true;
2287#endif
2288
2289
2290 kprobes_all_disarmed = false;
2291
2292 err = arch_init_kprobes();
2293 if (!err)
2294 err = register_die_notifier(&kprobe_exceptions_nb);
2295 if (!err)
2296 err = register_module_notifier(&kprobe_module_nb);
2297
2298 kprobes_initialized = (err == 0);
2299
2300 if (!err)
2301 init_test_probes();
2302 return err;
2303}
2304subsys_initcall(init_kprobes);
2305
2306#ifdef CONFIG_DEBUG_FS
2307static void report_probe(struct seq_file *pi, struct kprobe *p,
2308 const char *sym, int offset, char *modname, struct kprobe *pp)
2309{
2310 char *kprobe_type;
2311 void *addr = p->addr;
2312
2313 if (p->pre_handler == pre_handler_kretprobe)
2314 kprobe_type = "r";
2315 else
2316 kprobe_type = "k";
2317
2318 if (!kallsyms_show_value())
2319 addr = NULL;
2320
2321 if (sym)
2322 seq_printf(pi, "%px %s %s+0x%x %s ",
2323 addr, kprobe_type, sym, offset,
2324 (modname ? modname : " "));
2325 else
2326 seq_printf(pi, "%px %s %pS ",
2327 addr, kprobe_type, p->addr);
2328
2329 if (!pp)
2330 pp = p;
2331 seq_printf(pi, "%s%s%s%s\n",
2332 (kprobe_gone(p) ? "[GONE]" : ""),
2333 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
2334 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2335 (kprobe_ftrace(pp) ? "[FTRACE]" : ""));
2336}
2337
2338static void *kprobe_seq_start(struct seq_file *f, loff_t *pos)
2339{
2340 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2341}
2342
2343static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
2344{
2345 (*pos)++;
2346 if (*pos >= KPROBE_TABLE_SIZE)
2347 return NULL;
2348 return pos;
2349}
2350
2351static void kprobe_seq_stop(struct seq_file *f, void *v)
2352{
2353
2354}
2355
2356static int show_kprobe_addr(struct seq_file *pi, void *v)
2357{
2358 struct hlist_head *head;
2359 struct kprobe *p, *kp;
2360 const char *sym = NULL;
2361 unsigned int i = *(loff_t *) v;
2362 unsigned long offset = 0;
2363 char *modname, namebuf[KSYM_NAME_LEN];
2364
2365 head = &kprobe_table[i];
2366 preempt_disable();
2367 hlist_for_each_entry_rcu(p, head, hlist) {
2368 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
2369 &offset, &modname, namebuf);
2370 if (kprobe_aggrprobe(p)) {
2371 list_for_each_entry_rcu(kp, &p->list, list)
2372 report_probe(pi, kp, sym, offset, modname, p);
2373 } else
2374 report_probe(pi, p, sym, offset, modname, NULL);
2375 }
2376 preempt_enable();
2377 return 0;
2378}
2379
2380static const struct seq_operations kprobes_seq_ops = {
2381 .start = kprobe_seq_start,
2382 .next = kprobe_seq_next,
2383 .stop = kprobe_seq_stop,
2384 .show = show_kprobe_addr
2385};
2386
2387static int kprobes_open(struct inode *inode, struct file *filp)
2388{
2389 return seq_open(filp, &kprobes_seq_ops);
2390}
2391
2392static const struct file_operations debugfs_kprobes_operations = {
2393 .open = kprobes_open,
2394 .read = seq_read,
2395 .llseek = seq_lseek,
2396 .release = seq_release,
2397};
2398
2399
2400static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
2401{
2402 return seq_list_start(&kprobe_blacklist, *pos);
2403}
2404
2405static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos)
2406{
2407 return seq_list_next(v, &kprobe_blacklist, pos);
2408}
2409
2410static int kprobe_blacklist_seq_show(struct seq_file *m, void *v)
2411{
2412 struct kprobe_blacklist_entry *ent =
2413 list_entry(v, struct kprobe_blacklist_entry, list);
2414
2415
2416
2417
2418
2419 if (!kallsyms_show_value())
2420 seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL,
2421 (void *)ent->start_addr);
2422 else
2423 seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr,
2424 (void *)ent->end_addr, (void *)ent->start_addr);
2425 return 0;
2426}
2427
2428static const struct seq_operations kprobe_blacklist_seq_ops = {
2429 .start = kprobe_blacklist_seq_start,
2430 .next = kprobe_blacklist_seq_next,
2431 .stop = kprobe_seq_stop,
2432 .show = kprobe_blacklist_seq_show,
2433};
2434
2435static int kprobe_blacklist_open(struct inode *inode, struct file *filp)
2436{
2437 return seq_open(filp, &kprobe_blacklist_seq_ops);
2438}
2439
2440static const struct file_operations debugfs_kprobe_blacklist_ops = {
2441 .open = kprobe_blacklist_open,
2442 .read = seq_read,
2443 .llseek = seq_lseek,
2444 .release = seq_release,
2445};
2446
2447static int arm_all_kprobes(void)
2448{
2449 struct hlist_head *head;
2450 struct kprobe *p;
2451 unsigned int i, total = 0, errors = 0;
2452 int err, ret = 0;
2453
2454 mutex_lock(&kprobe_mutex);
2455
2456
2457 if (!kprobes_all_disarmed)
2458 goto already_enabled;
2459
2460
2461
2462
2463
2464
2465 kprobes_all_disarmed = false;
2466
2467 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2468 head = &kprobe_table[i];
2469
2470 hlist_for_each_entry_rcu(p, head, hlist) {
2471 if (!kprobe_disabled(p)) {
2472 err = arm_kprobe(p);
2473 if (err) {
2474 errors++;
2475 ret = err;
2476 }
2477 total++;
2478 }
2479 }
2480 }
2481
2482 if (errors)
2483 pr_warn("Kprobes globally enabled, but failed to arm %d out of %d probes\n",
2484 errors, total);
2485 else
2486 pr_info("Kprobes globally enabled\n");
2487
2488already_enabled:
2489 mutex_unlock(&kprobe_mutex);
2490 return ret;
2491}
2492
2493static int disarm_all_kprobes(void)
2494{
2495 struct hlist_head *head;
2496 struct kprobe *p;
2497 unsigned int i, total = 0, errors = 0;
2498 int err, ret = 0;
2499
2500 mutex_lock(&kprobe_mutex);
2501
2502
2503 if (kprobes_all_disarmed) {
2504 mutex_unlock(&kprobe_mutex);
2505 return 0;
2506 }
2507
2508 kprobes_all_disarmed = true;
2509
2510 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2511 head = &kprobe_table[i];
2512
2513 hlist_for_each_entry_rcu(p, head, hlist) {
2514 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
2515 err = disarm_kprobe(p, false);
2516 if (err) {
2517 errors++;
2518 ret = err;
2519 }
2520 total++;
2521 }
2522 }
2523 }
2524
2525 if (errors)
2526 pr_warn("Kprobes globally disabled, but failed to disarm %d out of %d probes\n",
2527 errors, total);
2528 else
2529 pr_info("Kprobes globally disabled\n");
2530
2531 mutex_unlock(&kprobe_mutex);
2532
2533
2534 wait_for_kprobe_optimizer();
2535
2536 return ret;
2537}
2538
2539
2540
2541
2542
2543
2544static ssize_t read_enabled_file_bool(struct file *file,
2545 char __user *user_buf, size_t count, loff_t *ppos)
2546{
2547 char buf[3];
2548
2549 if (!kprobes_all_disarmed)
2550 buf[0] = '1';
2551 else
2552 buf[0] = '0';
2553 buf[1] = '\n';
2554 buf[2] = 0x00;
2555 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2556}
2557
2558static ssize_t write_enabled_file_bool(struct file *file,
2559 const char __user *user_buf, size_t count, loff_t *ppos)
2560{
2561 char buf[32];
2562 size_t buf_size;
2563 int ret = 0;
2564
2565 buf_size = min(count, (sizeof(buf)-1));
2566 if (copy_from_user(buf, user_buf, buf_size))
2567 return -EFAULT;
2568
2569 buf[buf_size] = '\0';
2570 switch (buf[0]) {
2571 case 'y':
2572 case 'Y':
2573 case '1':
2574 ret = arm_all_kprobes();
2575 break;
2576 case 'n':
2577 case 'N':
2578 case '0':
2579 ret = disarm_all_kprobes();
2580 break;
2581 default:
2582 return -EINVAL;
2583 }
2584
2585 if (ret)
2586 return ret;
2587
2588 return count;
2589}
2590
2591static const struct file_operations fops_kp = {
2592 .read = read_enabled_file_bool,
2593 .write = write_enabled_file_bool,
2594 .llseek = default_llseek,
2595};
2596
2597static int __init debugfs_kprobe_init(void)
2598{
2599 struct dentry *dir;
2600 unsigned int value = 1;
2601
2602 dir = debugfs_create_dir("kprobes", NULL);
2603
2604 debugfs_create_file("list", 0400, dir, NULL,
2605 &debugfs_kprobes_operations);
2606
2607 debugfs_create_file("enabled", 0600, dir, &value, &fops_kp);
2608
2609 debugfs_create_file("blacklist", 0400, dir, NULL,
2610 &debugfs_kprobe_blacklist_ops);
2611
2612 return 0;
2613}
2614
2615late_initcall(debugfs_kprobe_init);
2616#endif
2617