1
2
3
4
5
6
7#ifndef _LINUX_BPF_H
8#define _LINUX_BPF_H 1
9
10#include <uapi/linux/bpf.h>
11#include <linux/workqueue.h>
12#include <linux/file.h>
13#include <linux/err.h>
14#include <linux/percpu.h>
15#include <linux/err.h>
16
17struct perf_event;
18struct bpf_map;
19
20
21struct bpf_map_ops {
22
23 int (*map_alloc_check)(union bpf_attr *attr);
24 struct bpf_map *(*map_alloc)(union bpf_attr *attr);
25 void (*map_release)(struct bpf_map *map, struct file *map_file);
26 void (*map_free)(struct bpf_map *map);
27 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
28
29
30 void *(*map_lookup_elem)(struct bpf_map *map, void *key);
31 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
32 int (*map_delete_elem)(struct bpf_map *map, void *key);
33
34
35 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
36 int fd);
37 void (*map_fd_put_ptr)(void *ptr);
38 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
39 u32 (*map_fd_sys_lookup_elem)(void *ptr);
40};
41
42struct bpf_map {
43
44
45
46 const struct bpf_map_ops *ops ____cacheline_aligned;
47 struct bpf_map *inner_map_meta;
48#ifdef CONFIG_SECURITY
49 void *security;
50#endif
51 enum bpf_map_type map_type;
52 u32 key_size;
53 u32 value_size;
54 u32 max_entries;
55 u32 map_flags;
56 u32 pages;
57 u32 id;
58 int numa_node;
59 bool unpriv_array;
60
61
62
63
64
65 struct user_struct *user ____cacheline_aligned;
66 atomic_t refcnt;
67 atomic_t usercnt;
68 struct work_struct work;
69 char name[BPF_OBJ_NAME_LEN];
70};
71
72
73enum bpf_arg_type {
74 ARG_DONTCARE = 0,
75
76
77
78
79 ARG_CONST_MAP_PTR,
80 ARG_PTR_TO_MAP_KEY,
81 ARG_PTR_TO_MAP_VALUE,
82
83
84
85
86 ARG_PTR_TO_MEM,
87 ARG_PTR_TO_MEM_OR_NULL,
88 ARG_PTR_TO_UNINIT_MEM,
89
90
91
92
93 ARG_CONST_SIZE,
94 ARG_CONST_SIZE_OR_ZERO,
95
96 ARG_PTR_TO_CTX,
97 ARG_ANYTHING,
98};
99
100
101enum bpf_return_type {
102 RET_INTEGER,
103 RET_VOID,
104 RET_PTR_TO_MAP_VALUE_OR_NULL,
105};
106
107
108
109
110
111struct bpf_func_proto {
112 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
113 bool gpl_only;
114 bool pkt_access;
115 enum bpf_return_type ret_type;
116 enum bpf_arg_type arg1_type;
117 enum bpf_arg_type arg2_type;
118 enum bpf_arg_type arg3_type;
119 enum bpf_arg_type arg4_type;
120 enum bpf_arg_type arg5_type;
121};
122
123
124
125
126
127struct bpf_context;
128
129enum bpf_access_type {
130 BPF_READ = 1,
131 BPF_WRITE = 2
132};
133
134
135
136
137
138
139
140
141
142
143
144enum bpf_reg_type {
145 NOT_INIT = 0,
146 SCALAR_VALUE,
147 PTR_TO_CTX,
148 CONST_PTR_TO_MAP,
149 PTR_TO_MAP_VALUE,
150 PTR_TO_MAP_VALUE_OR_NULL,
151 PTR_TO_STACK,
152 PTR_TO_PACKET_META,
153 PTR_TO_PACKET,
154 PTR_TO_PACKET_END,
155};
156
157struct bpf_prog;
158
159
160
161
162struct bpf_insn_access_aux {
163 enum bpf_reg_type reg_type;
164 int ctx_field_size;
165};
166
167static inline void
168bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
169{
170 aux->ctx_field_size = size;
171}
172
173struct bpf_prog_ops {
174 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
175 union bpf_attr __user *uattr);
176};
177
178struct bpf_verifier_ops {
179
180 const struct bpf_func_proto *
181 (*get_func_proto)(enum bpf_func_id func_id,
182 const struct bpf_prog *prog);
183
184
185
186
187 bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
188 const struct bpf_prog *prog,
189 struct bpf_insn_access_aux *info);
190 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
191 const struct bpf_prog *prog);
192 u32 (*convert_ctx_access)(enum bpf_access_type type,
193 const struct bpf_insn *src,
194 struct bpf_insn *dst,
195 struct bpf_prog *prog, u32 *target_size);
196};
197
198struct bpf_prog_aux {
199 atomic_t refcnt;
200 u32 used_map_cnt;
201 u32 id;
202 u32 func_cnt;
203 struct bpf_prog **func;
204 void *jit_data;
205 u32 max_ctx_offset;
206 u32 stack_depth;
207 const struct bpf_prog_ops *ops;
208 struct bpf_map **used_maps;
209 struct bpf_prog *prog;
210 struct user_struct *user;
211 u64 load_time;
212 char name[BPF_OBJ_NAME_LEN];
213#ifdef CONFIG_SECURITY
214 void *security;
215#endif
216 union {
217 struct work_struct work;
218 struct rcu_head rcu;
219 };
220};
221
222struct bpf_array {
223 struct bpf_map map;
224 u32 elem_size;
225 u32 index_mask;
226
227
228
229
230
231 enum bpf_prog_type owner_prog_type;
232 bool owner_jited;
233 union {
234 char value[0] __aligned(8);
235 void *ptrs[0] __aligned(8);
236 void __percpu *pptrs[0] __aligned(8);
237 };
238};
239
240#define MAX_TAIL_CALL_CNT 32
241
242struct bpf_event_entry {
243 struct perf_event *event;
244 struct file *perf_file;
245 struct file *map_file;
246 struct rcu_head rcu;
247};
248
249bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
250int bpf_prog_calc_tag(struct bpf_prog *fp);
251
252const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
253
254typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
255 unsigned long off, unsigned long len);
256
257u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
258 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
259
260int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
261 union bpf_attr __user *uattr);
262
263
264
265
266
267
268
269
270
271
272
273
274
275struct bpf_prog_array {
276 struct rcu_head rcu;
277 struct bpf_prog *progs[0];
278};
279
280struct bpf_prog_array __rcu *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
281void bpf_prog_array_free(struct bpf_prog_array __rcu *progs);
282int bpf_prog_array_length(struct bpf_prog_array __rcu *progs);
283int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
284 __u32 __user *prog_ids, u32 cnt);
285
286void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *progs,
287 struct bpf_prog *old_prog);
288int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
289 u32 *prog_ids, u32 request_cnt,
290 u32 *prog_cnt);
291int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
292 struct bpf_prog *exclude_prog,
293 struct bpf_prog *include_prog,
294 struct bpf_prog_array **new_array);
295
296#define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \
297 ({ \
298 struct bpf_prog **_prog, *__prog; \
299 struct bpf_prog_array *_array; \
300 u32 _ret = 1; \
301 rcu_read_lock(); \
302 _array = rcu_dereference(array); \
303 if (unlikely(check_non_null && !_array))\
304 goto _out; \
305 _prog = _array->progs; \
306 while ((__prog = READ_ONCE(*_prog))) { \
307 _ret &= func(__prog, ctx); \
308 _prog++; \
309 } \
310_out: \
311 rcu_read_unlock(); \
312 _ret; \
313 })
314
315#define BPF_PROG_RUN_ARRAY(array, ctx, func) \
316 __BPF_PROG_RUN_ARRAY(array, ctx, func, false)
317
318#define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \
319 __BPF_PROG_RUN_ARRAY(array, ctx, func, true)
320
321#ifdef CONFIG_BPF_SYSCALL
322DECLARE_PER_CPU(int, bpf_prog_active);
323
324extern const struct file_operations bpf_map_fops;
325extern const struct file_operations bpf_prog_fops;
326
327#define BPF_PROG_TYPE(_id, _name) \
328 extern const struct bpf_prog_ops _name ## _prog_ops; \
329 extern const struct bpf_verifier_ops _name ## _verifier_ops;
330#define BPF_MAP_TYPE(_id, _ops) \
331 extern const struct bpf_map_ops _ops;
332#include <linux/bpf_types.h>
333#undef BPF_PROG_TYPE
334#undef BPF_MAP_TYPE
335
336struct bpf_prog *bpf_prog_get(u32 ufd);
337struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type);
338struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i);
339struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog);
340struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
341void bpf_prog_put(struct bpf_prog *prog);
342int __bpf_prog_charge(struct user_struct *user, u32 pages);
343void __bpf_prog_uncharge(struct user_struct *user, u32 pages);
344
345struct bpf_map *bpf_map_get(u32 ufd);
346struct bpf_map *__bpf_map_get(struct fd f);
347struct bpf_map *bpf_map_get_with_uref(u32 ufd);
348struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref);
349void bpf_map_put_with_uref(struct bpf_map *map);
350void bpf_map_put(struct bpf_map *map);
351int bpf_map_precharge_memlock(u32 pages);
352void *bpf_map_area_alloc(size_t size, int numa_node);
353void bpf_map_area_free(void *base);
354void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
355
356extern int sysctl_unprivileged_bpf_disabled;
357
358int bpf_map_new_fd(struct bpf_map *map, int flags);
359int bpf_prog_new_fd(struct bpf_prog *prog);
360
361int bpf_obj_pin_user(u32 ufd, const char __user *pathname);
362int bpf_obj_get_user(const char __user *pathname, int flags);
363
364int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
365int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
366int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
367 u64 flags);
368int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
369 u64 flags);
370
371int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
372
373int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
374 void *key, void *value, u64 map_flags);
375int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
376void bpf_fd_array_map_clear(struct bpf_map *map);
377int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
378 void *key, void *value, u64 map_flags);
379int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
380
381int bpf_get_file_flag(int flags);
382
383
384
385
386
387
388
389static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
390{
391 const long *lsrc = src;
392 long *ldst = dst;
393
394 size /= sizeof(long);
395 while (size--)
396 *ldst++ = *lsrc++;
397}
398
399
400int bpf_check(struct bpf_prog **fp, union bpf_attr *attr);
401void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
402
403
404static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
405{
406 return (attr->map_flags & BPF_F_NUMA_NODE) ?
407 attr->numa_node : NUMA_NO_NODE;
408}
409
410#else
411static inline struct bpf_prog *bpf_prog_get(u32 ufd)
412{
413 return ERR_PTR(-EOPNOTSUPP);
414}
415
416static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
417 enum bpf_prog_type type)
418{
419 return ERR_PTR(-EOPNOTSUPP);
420}
421static inline struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog,
422 int i)
423{
424 return ERR_PTR(-EOPNOTSUPP);
425}
426
427static inline void bpf_prog_put(struct bpf_prog *prog)
428{
429}
430
431static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog)
432{
433 return ERR_PTR(-EOPNOTSUPP);
434}
435
436static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
437{
438}
439
440static inline struct bpf_prog *__must_check
441bpf_prog_inc_not_zero(struct bpf_prog *prog)
442{
443 return ERR_PTR(-EOPNOTSUPP);
444}
445
446static inline int __bpf_prog_charge(struct user_struct *user, u32 pages)
447{
448 return 0;
449}
450
451static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
452{
453}
454
455static inline int bpf_obj_get_user(const char __user *pathname, int flags)
456{
457 return -EOPNOTSUPP;
458}
459
460#endif
461
462
463extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
464extern const struct bpf_func_proto bpf_map_update_elem_proto;
465extern const struct bpf_func_proto bpf_map_delete_elem_proto;
466
467extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
468extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
469extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
470extern const struct bpf_func_proto bpf_tail_call_proto;
471extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
472extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
473extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
474extern const struct bpf_func_proto bpf_get_current_comm_proto;
475extern const struct bpf_func_proto bpf_get_stackid_proto;
476
477
478void bpf_user_rnd_init_once(void);
479u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
480
481bool bpf_helper_changes_skb_data(void *func);
482#endif
483