1
2#ifndef _LINUX_KPROBES_H
3#define _LINUX_KPROBES_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/compiler.h>
19#include <linux/linkage.h>
20#include <linux/list.h>
21#include <linux/notifier.h>
22#include <linux/smp.h>
23#include <linux/bug.h>
24#include <linux/percpu.h>
25#include <linux/spinlock.h>
26#include <linux/rcupdate.h>
27#include <linux/mutex.h>
28#include <linux/ftrace.h>
29#include <linux/refcount.h>
30#include <linux/freelist.h>
31#include <asm/kprobes.h>
32
33#ifdef CONFIG_KPROBES
34
35
36#define KPROBE_HIT_ACTIVE 0x00000001
37#define KPROBE_HIT_SS 0x00000002
38#define KPROBE_REENTER 0x00000004
39#define KPROBE_HIT_SSDONE 0x00000008
40
41#else
42#include <asm-generic/kprobes.h>
43typedef int kprobe_opcode_t;
44struct arch_specific_insn {
45 int dummy;
46};
47#endif
48
49struct kprobe;
50struct pt_regs;
51struct kretprobe;
52struct kretprobe_instance;
53typedef int (*kprobe_pre_handler_t) (struct kprobe *, struct pt_regs *);
54typedef void (*kprobe_post_handler_t) (struct kprobe *, struct pt_regs *,
55 unsigned long flags);
56typedef int (*kretprobe_handler_t) (struct kretprobe_instance *,
57 struct pt_regs *);
58
59struct kprobe {
60 struct hlist_node hlist;
61
62
63 struct list_head list;
64
65
66 unsigned long nmissed;
67
68
69 kprobe_opcode_t *addr;
70
71
72 const char *symbol_name;
73
74
75 unsigned int offset;
76
77
78 kprobe_pre_handler_t pre_handler;
79
80
81 kprobe_post_handler_t post_handler;
82
83
84 kprobe_opcode_t opcode;
85
86
87 struct arch_specific_insn ainsn;
88
89
90
91
92
93 u32 flags;
94};
95
96
97#define KPROBE_FLAG_GONE 1
98#define KPROBE_FLAG_DISABLED 2
99#define KPROBE_FLAG_OPTIMIZED 4
100
101
102
103
104#define KPROBE_FLAG_FTRACE 8
105
106
107static inline bool kprobe_gone(struct kprobe *p)
108{
109 return p->flags & KPROBE_FLAG_GONE;
110}
111
112
113static inline bool kprobe_disabled(struct kprobe *p)
114{
115 return p->flags & (KPROBE_FLAG_DISABLED | KPROBE_FLAG_GONE);
116}
117
118
119static inline bool kprobe_optimized(struct kprobe *p)
120{
121 return p->flags & KPROBE_FLAG_OPTIMIZED;
122}
123
124
125static inline bool kprobe_ftrace(struct kprobe *p)
126{
127 return p->flags & KPROBE_FLAG_FTRACE;
128}
129
130
131
132
133
134
135
136
137
138
139
140struct kretprobe_holder {
141 struct kretprobe *rp;
142 refcount_t ref;
143};
144
145struct kretprobe {
146 struct kprobe kp;
147 kretprobe_handler_t handler;
148 kretprobe_handler_t entry_handler;
149 int maxactive;
150 int nmissed;
151 size_t data_size;
152 struct freelist_head freelist;
153 struct kretprobe_holder *rph;
154};
155
156#define KRETPROBE_MAX_DATA_SIZE 4096
157
158struct kretprobe_instance {
159 union {
160 struct freelist_node freelist;
161 struct rcu_head rcu;
162 };
163 struct llist_node llist;
164 struct kretprobe_holder *rph;
165 kprobe_opcode_t *ret_addr;
166 void *fp;
167 char data[];
168};
169
170struct kretprobe_blackpoint {
171 const char *name;
172 void *addr;
173};
174
175struct kprobe_blacklist_entry {
176 struct list_head list;
177 unsigned long start_addr;
178 unsigned long end_addr;
179};
180
181#ifdef CONFIG_KPROBES
182DECLARE_PER_CPU(struct kprobe *, current_kprobe);
183DECLARE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
184
185extern void kprobe_busy_begin(void);
186extern void kprobe_busy_end(void);
187
188#ifdef CONFIG_KRETPROBES
189extern void arch_prepare_kretprobe(struct kretprobe_instance *ri,
190 struct pt_regs *regs);
191extern int arch_trampoline_kprobe(struct kprobe *p);
192
193void arch_kretprobe_fixup_return(struct pt_regs *regs,
194 kprobe_opcode_t *correct_ret_addr);
195
196void __kretprobe_trampoline(void);
197
198
199
200
201static nokprobe_inline void *kretprobe_trampoline_addr(void)
202{
203 return dereference_kernel_function_descriptor(__kretprobe_trampoline);
204}
205
206
207unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
208 void *frame_pointer);
209
210static nokprobe_inline
211unsigned long kretprobe_trampoline_handler(struct pt_regs *regs,
212 void *frame_pointer)
213{
214 unsigned long ret;
215
216
217
218
219
220 kprobe_busy_begin();
221 ret = __kretprobe_trampoline_handler(regs, frame_pointer);
222 kprobe_busy_end();
223
224 return ret;
225}
226
227static nokprobe_inline struct kretprobe *get_kretprobe(struct kretprobe_instance *ri)
228{
229 RCU_LOCKDEP_WARN(!rcu_read_lock_any_held(),
230 "Kretprobe is accessed from instance under preemptive context");
231
232 return READ_ONCE(ri->rph->rp);
233}
234
235#else
236static inline void arch_prepare_kretprobe(struct kretprobe *rp,
237 struct pt_regs *regs)
238{
239}
240static inline int arch_trampoline_kprobe(struct kprobe *p)
241{
242 return 0;
243}
244#endif
245
246
247extern unsigned long __start_kprobe_blacklist[];
248extern unsigned long __stop_kprobe_blacklist[];
249
250extern struct kretprobe_blackpoint kretprobe_blacklist[];
251
252#ifdef CONFIG_KPROBES_SANITY_TEST
253extern int init_test_probes(void);
254#else
255static inline int init_test_probes(void)
256{
257 return 0;
258}
259#endif
260
261extern int arch_prepare_kprobe(struct kprobe *p);
262extern void arch_arm_kprobe(struct kprobe *p);
263extern void arch_disarm_kprobe(struct kprobe *p);
264extern int arch_init_kprobes(void);
265extern void kprobes_inc_nmissed_count(struct kprobe *p);
266extern bool arch_within_kprobe_blacklist(unsigned long addr);
267extern int arch_populate_kprobe_blacklist(void);
268extern bool arch_kprobe_on_func_entry(unsigned long offset);
269extern int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset);
270
271extern bool within_kprobe_blacklist(unsigned long addr);
272extern int kprobe_add_ksym_blacklist(unsigned long entry);
273extern int kprobe_add_area_blacklist(unsigned long start, unsigned long end);
274
275struct kprobe_insn_cache {
276 struct mutex mutex;
277 void *(*alloc)(void);
278 void (*free)(void *);
279 const char *sym;
280 struct list_head pages;
281 size_t insn_size;
282 int nr_garbage;
283};
284
285#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
286extern kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c);
287extern void __free_insn_slot(struct kprobe_insn_cache *c,
288 kprobe_opcode_t *slot, int dirty);
289
290extern bool __is_insn_slot_addr(struct kprobe_insn_cache *c,
291 unsigned long addr);
292
293#define DEFINE_INSN_CACHE_OPS(__name) \
294extern struct kprobe_insn_cache kprobe_##__name##_slots; \
295 \
296static inline kprobe_opcode_t *get_##__name##_slot(void) \
297{ \
298 return __get_insn_slot(&kprobe_##__name##_slots); \
299} \
300 \
301static inline void free_##__name##_slot(kprobe_opcode_t *slot, int dirty)\
302{ \
303 __free_insn_slot(&kprobe_##__name##_slots, slot, dirty); \
304} \
305 \
306static inline bool is_kprobe_##__name##_slot(unsigned long addr) \
307{ \
308 return __is_insn_slot_addr(&kprobe_##__name##_slots, addr); \
309}
310#define KPROBE_INSN_PAGE_SYM "kprobe_insn_page"
311#define KPROBE_OPTINSN_PAGE_SYM "kprobe_optinsn_page"
312int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum,
313 unsigned long *value, char *type, char *sym);
314#else
315#define DEFINE_INSN_CACHE_OPS(__name) \
316static inline bool is_kprobe_##__name##_slot(unsigned long addr) \
317{ \
318 return 0; \
319}
320#endif
321
322DEFINE_INSN_CACHE_OPS(insn);
323
324#ifdef CONFIG_OPTPROBES
325
326
327
328struct optimized_kprobe {
329 struct kprobe kp;
330 struct list_head list;
331 struct arch_optimized_insn optinsn;
332};
333
334
335extern int arch_prepared_optinsn(struct arch_optimized_insn *optinsn);
336extern int arch_check_optimized_kprobe(struct optimized_kprobe *op);
337extern int arch_prepare_optimized_kprobe(struct optimized_kprobe *op,
338 struct kprobe *orig);
339extern void arch_remove_optimized_kprobe(struct optimized_kprobe *op);
340extern void arch_optimize_kprobes(struct list_head *oplist);
341extern void arch_unoptimize_kprobes(struct list_head *oplist,
342 struct list_head *done_list);
343extern void arch_unoptimize_kprobe(struct optimized_kprobe *op);
344extern int arch_within_optimized_kprobe(struct optimized_kprobe *op,
345 kprobe_opcode_t *addr);
346
347extern void opt_pre_handler(struct kprobe *p, struct pt_regs *regs);
348
349DEFINE_INSN_CACHE_OPS(optinsn);
350
351#ifdef CONFIG_SYSCTL
352extern int sysctl_kprobes_optimization;
353extern int proc_kprobes_optimization_handler(struct ctl_table *table,
354 int write, void *buffer,
355 size_t *length, loff_t *ppos);
356#endif
357extern void wait_for_kprobe_optimizer(void);
358#else
359static inline void wait_for_kprobe_optimizer(void) { }
360#endif
361
362#ifdef CONFIG_KPROBES_ON_FTRACE
363extern void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
364 struct ftrace_ops *ops, struct ftrace_regs *fregs);
365extern int arch_prepare_kprobe_ftrace(struct kprobe *p);
366#else
367static inline int arch_prepare_kprobe_ftrace(struct kprobe *p)
368{
369 return -EINVAL;
370}
371#endif
372
373
374struct kprobe *get_kprobe(void *addr);
375
376
377static inline struct kprobe *kprobe_running(void)
378{
379 return __this_cpu_read(current_kprobe);
380}
381
382static inline void reset_current_kprobe(void)
383{
384 __this_cpu_write(current_kprobe, NULL);
385}
386
387static inline struct kprobe_ctlblk *get_kprobe_ctlblk(void)
388{
389 return this_cpu_ptr(&kprobe_ctlblk);
390}
391
392kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset);
393int register_kprobe(struct kprobe *p);
394void unregister_kprobe(struct kprobe *p);
395int register_kprobes(struct kprobe **kps, int num);
396void unregister_kprobes(struct kprobe **kps, int num);
397
398int register_kretprobe(struct kretprobe *rp);
399void unregister_kretprobe(struct kretprobe *rp);
400int register_kretprobes(struct kretprobe **rps, int num);
401void unregister_kretprobes(struct kretprobe **rps, int num);
402
403void kprobe_flush_task(struct task_struct *tk);
404
405void kprobe_free_init_mem(void);
406
407int disable_kprobe(struct kprobe *kp);
408int enable_kprobe(struct kprobe *kp);
409
410void dump_kprobe(struct kprobe *kp);
411
412void *alloc_insn_page(void);
413
414void *alloc_optinsn_page(void);
415void free_optinsn_page(void *page);
416
417int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
418 char *sym);
419
420int arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value,
421 char *type, char *sym);
422#else
423
424static inline int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
425{
426 return 0;
427}
428static inline struct kprobe *get_kprobe(void *addr)
429{
430 return NULL;
431}
432static inline struct kprobe *kprobe_running(void)
433{
434 return NULL;
435}
436static inline int register_kprobe(struct kprobe *p)
437{
438 return -EOPNOTSUPP;
439}
440static inline int register_kprobes(struct kprobe **kps, int num)
441{
442 return -EOPNOTSUPP;
443}
444static inline void unregister_kprobe(struct kprobe *p)
445{
446}
447static inline void unregister_kprobes(struct kprobe **kps, int num)
448{
449}
450static inline int register_kretprobe(struct kretprobe *rp)
451{
452 return -EOPNOTSUPP;
453}
454static inline int register_kretprobes(struct kretprobe **rps, int num)
455{
456 return -EOPNOTSUPP;
457}
458static inline void unregister_kretprobe(struct kretprobe *rp)
459{
460}
461static inline void unregister_kretprobes(struct kretprobe **rps, int num)
462{
463}
464static inline void kprobe_flush_task(struct task_struct *tk)
465{
466}
467static inline void kprobe_free_init_mem(void)
468{
469}
470static inline int disable_kprobe(struct kprobe *kp)
471{
472 return -EOPNOTSUPP;
473}
474static inline int enable_kprobe(struct kprobe *kp)
475{
476 return -EOPNOTSUPP;
477}
478
479static inline bool within_kprobe_blacklist(unsigned long addr)
480{
481 return true;
482}
483static inline int kprobe_get_kallsym(unsigned int symnum, unsigned long *value,
484 char *type, char *sym)
485{
486 return -ERANGE;
487}
488#endif
489
490static inline int disable_kretprobe(struct kretprobe *rp)
491{
492 return disable_kprobe(&rp->kp);
493}
494static inline int enable_kretprobe(struct kretprobe *rp)
495{
496 return enable_kprobe(&rp->kp);
497}
498
499#ifndef CONFIG_KPROBES
500static inline bool is_kprobe_insn_slot(unsigned long addr)
501{
502 return false;
503}
504#endif
505
506#ifndef CONFIG_OPTPROBES
507static inline bool is_kprobe_optinsn_slot(unsigned long addr)
508{
509 return false;
510}
511#endif
512
513#ifdef CONFIG_KRETPROBES
514static nokprobe_inline bool is_kretprobe_trampoline(unsigned long addr)
515{
516 return (void *)addr == kretprobe_trampoline_addr();
517}
518
519unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp,
520 struct llist_node **cur);
521#else
522static nokprobe_inline bool is_kretprobe_trampoline(unsigned long addr)
523{
524 return false;
525}
526
527static nokprobe_inline
528unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp,
529 struct llist_node **cur)
530{
531 return 0;
532}
533#endif
534
535
536static nokprobe_inline bool kprobe_page_fault(struct pt_regs *regs,
537 unsigned int trap)
538{
539 if (!IS_ENABLED(CONFIG_KPROBES))
540 return false;
541 if (user_mode(regs))
542 return false;
543
544
545
546
547 if (preemptible())
548 return false;
549 if (!kprobe_running())
550 return false;
551 return kprobe_fault_handler(regs, trap);
552}
553
554#endif
555