1
2#ifndef _LINUX_KPROBES_H
3#define _LINUX_KPROBES_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/compiler.h>
20#include <linux/linkage.h>
21#include <linux/list.h>
22#include <linux/notifier.h>
23#include <linux/smp.h>
24#include <linux/bug.h>
25#include <linux/percpu.h>
26#include <linux/spinlock.h>
27#include <linux/rcupdate.h>
28#include <linux/mutex.h>
29#include <linux/ftrace.h>
30#include <linux/refcount.h>
31#include <linux/freelist.h>
32#include <asm/kprobes.h>
33
34#ifdef CONFIG_KPROBES
35
36
37#define KPROBE_HIT_ACTIVE 0x00000001
38#define KPROBE_HIT_SS 0x00000002
39#define KPROBE_REENTER 0x00000004
40#define KPROBE_HIT_SSDONE 0x00000008
41
42#else
43#include <asm-generic/kprobes.h>
44typedef int kprobe_opcode_t;
45struct arch_specific_insn {
46 int dummy;
47};
48#endif
49
50struct kprobe;
51struct pt_regs;
52struct kretprobe;
53struct kretprobe_instance;
54typedef int (*kprobe_pre_handler_t) (struct kprobe *, struct pt_regs *);
55typedef void (*kprobe_post_handler_t) (struct kprobe *, struct pt_regs *,
56 unsigned long flags);
57typedef int (*kprobe_fault_handler_t) (struct kprobe *, struct pt_regs *,
58 int trapnr);
59typedef int (*kretprobe_handler_t) (struct kretprobe_instance *,
60 struct pt_regs *);
61
62struct kprobe {
63 struct hlist_node hlist;
64
65
66 struct list_head list;
67
68
69 unsigned long nmissed;
70
71
72 kprobe_opcode_t *addr;
73
74
75 const char *symbol_name;
76
77
78 unsigned int offset;
79
80
81 kprobe_pre_handler_t pre_handler;
82
83
84 kprobe_post_handler_t post_handler;
85
86
87
88
89
90 kprobe_fault_handler_t fault_handler;
91
92
93 kprobe_opcode_t opcode;
94
95
96 struct arch_specific_insn ainsn;
97
98
99
100
101
102 u32 flags;
103};
104
105
106#define KPROBE_FLAG_GONE 1
107#define KPROBE_FLAG_DISABLED 2
108#define KPROBE_FLAG_OPTIMIZED 4
109
110
111
112
113#define KPROBE_FLAG_FTRACE 8
114
115
116static inline int kprobe_gone(struct kprobe *p)
117{
118 return p->flags & KPROBE_FLAG_GONE;
119}
120
121
122static inline int kprobe_disabled(struct kprobe *p)
123{
124 return p->flags & (KPROBE_FLAG_DISABLED | KPROBE_FLAG_GONE);
125}
126
127
128static inline int kprobe_optimized(struct kprobe *p)
129{
130 return p->flags & KPROBE_FLAG_OPTIMIZED;
131}
132
133
134static inline int kprobe_ftrace(struct kprobe *p)
135{
136 return p->flags & KPROBE_FLAG_FTRACE;
137}
138
139
140
141
142
143
144
145
146
147
148
149struct kretprobe_holder {
150 struct kretprobe *rp;
151 refcount_t ref;
152};
153
154struct kretprobe {
155 struct kprobe kp;
156 kretprobe_handler_t handler;
157 kretprobe_handler_t entry_handler;
158 int maxactive;
159 int nmissed;
160 size_t data_size;
161 struct freelist_head freelist;
162 struct kretprobe_holder *rph;
163};
164
165struct kretprobe_instance {
166 union {
167 struct freelist_node freelist;
168 struct rcu_head rcu;
169 };
170 struct llist_node llist;
171 struct kretprobe_holder *rph;
172 kprobe_opcode_t *ret_addr;
173 void *fp;
174 char data[];
175};
176
177struct kretprobe_blackpoint {
178 const char *name;
179 void *addr;
180};
181
182struct kprobe_blacklist_entry {
183 struct list_head list;
184 unsigned long start_addr;
185 unsigned long end_addr;
186};
187
188#ifdef CONFIG_KPROBES
189DECLARE_PER_CPU(struct kprobe *, current_kprobe);
190DECLARE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
191
192
193
194
195static inline int kprobes_built_in(void)
196{
197 return 1;
198}
199
200extern void kprobe_busy_begin(void);
201extern void kprobe_busy_end(void);
202
203#ifdef CONFIG_KRETPROBES
204extern void arch_prepare_kretprobe(struct kretprobe_instance *ri,
205 struct pt_regs *regs);
206extern int arch_trampoline_kprobe(struct kprobe *p);
207
208
209unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
210 void *trampoline_address,
211 void *frame_pointer);
212
213static nokprobe_inline
214unsigned long kretprobe_trampoline_handler(struct pt_regs *regs,
215 void *trampoline_address,
216 void *frame_pointer)
217{
218 unsigned long ret;
219
220
221
222
223
224 kprobe_busy_begin();
225 ret = __kretprobe_trampoline_handler(regs, trampoline_address, frame_pointer);
226 kprobe_busy_end();
227
228 return ret;
229}
230
231static nokprobe_inline struct kretprobe *get_kretprobe(struct kretprobe_instance *ri)
232{
233 RCU_LOCKDEP_WARN(!rcu_read_lock_any_held(),
234 "Kretprobe is accessed from instance under preemptive context");
235
236 return READ_ONCE(ri->rph->rp);
237}
238
239#else
240static inline void arch_prepare_kretprobe(struct kretprobe *rp,
241 struct pt_regs *regs)
242{
243}
244static inline int arch_trampoline_kprobe(struct kprobe *p)
245{
246 return 0;
247}
248#endif
249
250extern struct kretprobe_blackpoint kretprobe_blacklist[];
251
252#ifdef CONFIG_KPROBES_SANITY_TEST
253extern int init_test_probes(void);
254#else
255static inline int init_test_probes(void)
256{
257 return 0;
258}
259#endif
260
261extern int arch_prepare_kprobe(struct kprobe *p);
262extern void arch_arm_kprobe(struct kprobe *p);
263extern void arch_disarm_kprobe(struct kprobe *p);
264extern int arch_init_kprobes(void);
265extern void kprobes_inc_nmissed_count(struct kprobe *p);
266extern bool arch_within_kprobe_blacklist(unsigned long addr);
267extern int arch_populate_kprobe_blacklist(void);
268extern bool arch_kprobe_on_func_entry(unsigned long offset);
269extern int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset);
270
271extern bool within_kprobe_blacklist(unsigned long addr);
272extern int kprobe_add_ksym_blacklist(unsigned long entry);
273extern int kprobe_add_area_blacklist(unsigned long start, unsigned long end);
274
275struct kprobe_insn_cache {
276 struct mutex mutex;
277 void *(*alloc)(void);
278 void (*free)(void *);
279 const char *sym;
280 struct list_head pages;
281 size_t insn_size;
282 int nr_garbage;
283};
284
285#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
286extern kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c);
287extern void __free_insn_slot(struct kprobe_insn_cache *c,
288 kprobe_opcode_t *slot, int dirty);
289
290extern bool __is_insn_slot_addr(struct kprobe_insn_cache *c,
291 unsigned long addr);
292
293#define DEFINE_INSN_CACHE_OPS(__name) \
294extern struct kprobe_insn_cache kprobe_##__name##_slots; \
295 \
296static inline kprobe_opcode_t *get_##__name##_slot(void) \
297{ \
298 return __get_insn_slot(&kprobe_##__name##_slots); \
299} \
300 \
301static inline void free_##__name##_slot(kprobe_opcode_t *slot, int dirty)\
302{ \
303 __free_insn_slot(&kprobe_##__name##_slots, slot, dirty); \
304} \
305 \
306static inline bool is_kprobe_##__name##_slot(unsigned long addr) \
307{ \
308 return __is_insn_slot_addr(&kprobe_##__name##_slots, addr); \
309}
310#define KPROBE_INSN_PAGE_SYM "kprobe_insn_page"
311#define KPROBE_OPTINSN_PAGE_SYM "kprobe_optinsn_page"
312int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum,
313 unsigned long *value, char *type, char *sym);
314#else
315#define DEFINE_INSN_CACHE_OPS(__name) \
316static inline bool is_kprobe_##__name##_slot(unsigned long addr) \
317{ \
318 return 0; \
319}
320#endif
321
322DEFINE_INSN_CACHE_OPS(insn);
323
324#ifdef CONFIG_OPTPROBES
325
326
327
328struct optimized_kprobe {
329 struct kprobe kp;
330 struct list_head list;
331 struct arch_optimized_insn optinsn;
332};
333
334
335extern int arch_prepared_optinsn(struct arch_optimized_insn *optinsn);
336extern int arch_check_optimized_kprobe(struct optimized_kprobe *op);
337extern int arch_prepare_optimized_kprobe(struct optimized_kprobe *op,
338 struct kprobe *orig);
339extern void arch_remove_optimized_kprobe(struct optimized_kprobe *op);
340extern void arch_optimize_kprobes(struct list_head *oplist);
341extern void arch_unoptimize_kprobes(struct list_head *oplist,
342 struct list_head *done_list);
343extern void arch_unoptimize_kprobe(struct optimized_kprobe *op);
344extern int arch_within_optimized_kprobe(struct optimized_kprobe *op,
345 unsigned long addr);
346
347extern void opt_pre_handler(struct kprobe *p, struct pt_regs *regs);
348
349DEFINE_INSN_CACHE_OPS(optinsn);
350
351#ifdef CONFIG_SYSCTL
352extern int sysctl_kprobes_optimization;
353extern int proc_kprobes_optimization_handler(struct ctl_table *table,
354 int write, void *buffer,
355 size_t *length, loff_t *ppos);
356#endif
357extern void wait_for_kprobe_optimizer(void);
358#else
359static inline void wait_for_kprobe_optimizer(void) { }
360#endif
361#ifdef CONFIG_KPROBES_ON_FTRACE
362extern void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
363 struct ftrace_ops *ops, struct ftrace_regs *fregs);
364extern int arch_prepare_kprobe_ftrace(struct kprobe *p);
365#endif
366
367int arch_check_ftrace_location(struct kprobe *p);
368
369
370struct kprobe *get_kprobe(void *addr);
371
372
373static inline struct kprobe *kprobe_running(void)
374{
375 return (__this_cpu_read(current_kprobe));
376}
377
378static inline void reset_current_kprobe(void)
379{
380 __this_cpu_write(current_kprobe, NULL);
381}
382
383static inline struct kprobe_ctlblk *get_kprobe_ctlblk(void)
384{
385 return this_cpu_ptr(&kprobe_ctlblk);
386}
387
388kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset);
389int register_kprobe(struct kprobe *p);
390void unregister_kprobe(struct kprobe *p);
391int register_kprobes(struct kprobe **kps, int num);
392void unregister_kprobes(struct kprobe **kps, int num);
393unsigned long arch_deref_entry_point(void *);
394
395int register_kretprobe(struct kretprobe *rp);
396void unregister_kretprobe(struct kretprobe *rp);
397int register_kretprobes(struct kretprobe **rps, int num);
398void unregister_kretprobes(struct kretprobe **rps, int num);
399
400void kprobe_flush_task(struct task_struct *tk);
401
402void kprobe_free_init_mem(void);
403
404int disable_kprobe(struct kprobe *kp);
405int enable_kprobe(struct kprobe *kp);
406
407void dump_kprobe(struct kprobe *kp);
408
409void *alloc_insn_page(void);
410void free_insn_page(void *page);
411
412int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
413 char *sym);
414
415int arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value,
416 char *type, char *sym);
417#else
418
419static inline int kprobes_built_in(void)
420{
421 return 0;
422}
423static inline int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
424{
425 return 0;
426}
427static inline struct kprobe *get_kprobe(void *addr)
428{
429 return NULL;
430}
431static inline struct kprobe *kprobe_running(void)
432{
433 return NULL;
434}
435static inline int register_kprobe(struct kprobe *p)
436{
437 return -ENOSYS;
438}
439static inline int register_kprobes(struct kprobe **kps, int num)
440{
441 return -ENOSYS;
442}
443static inline void unregister_kprobe(struct kprobe *p)
444{
445}
446static inline void unregister_kprobes(struct kprobe **kps, int num)
447{
448}
449static inline int register_kretprobe(struct kretprobe *rp)
450{
451 return -ENOSYS;
452}
453static inline int register_kretprobes(struct kretprobe **rps, int num)
454{
455 return -ENOSYS;
456}
457static inline void unregister_kretprobe(struct kretprobe *rp)
458{
459}
460static inline void unregister_kretprobes(struct kretprobe **rps, int num)
461{
462}
463static inline void kprobe_flush_task(struct task_struct *tk)
464{
465}
466static inline void kprobe_free_init_mem(void)
467{
468}
469static inline int disable_kprobe(struct kprobe *kp)
470{
471 return -ENOSYS;
472}
473static inline int enable_kprobe(struct kprobe *kp)
474{
475 return -ENOSYS;
476}
477
478static inline bool within_kprobe_blacklist(unsigned long addr)
479{
480 return true;
481}
482static inline int kprobe_get_kallsym(unsigned int symnum, unsigned long *value,
483 char *type, char *sym)
484{
485 return -ERANGE;
486}
487#endif
488static inline int disable_kretprobe(struct kretprobe *rp)
489{
490 return disable_kprobe(&rp->kp);
491}
492static inline int enable_kretprobe(struct kretprobe *rp)
493{
494 return enable_kprobe(&rp->kp);
495}
496
497#ifndef CONFIG_KPROBES
498static inline bool is_kprobe_insn_slot(unsigned long addr)
499{
500 return false;
501}
502#endif
503#ifndef CONFIG_OPTPROBES
504static inline bool is_kprobe_optinsn_slot(unsigned long addr)
505{
506 return false;
507}
508#endif
509
510
511static nokprobe_inline bool kprobe_page_fault(struct pt_regs *regs,
512 unsigned int trap)
513{
514 if (!kprobes_built_in())
515 return false;
516 if (user_mode(regs))
517 return false;
518
519
520
521
522 if (preemptible())
523 return false;
524 if (!kprobe_running())
525 return false;
526 return kprobe_fault_handler(regs, trap);
527}
528
529#endif
530