1
2
3
4#ifndef _LINUX_BPF_H
5#define _LINUX_BPF_H 1
6
7#include <uapi/linux/bpf.h>
8
9#include <linux/workqueue.h>
10#include <linux/file.h>
11#include <linux/percpu.h>
12#include <linux/err.h>
13#include <linux/rbtree_latch.h>
14#include <linux/numa.h>
15#include <linux/mm_types.h>
16#include <linux/wait.h>
17#include <linux/u64_stats_sync.h>
18#include <linux/refcount.h>
19#include <linux/mutex.h>
20#include <linux/module.h>
21
22struct bpf_verifier_env;
23struct bpf_verifier_log;
24struct perf_event;
25struct bpf_prog;
26struct bpf_prog_aux;
27struct bpf_map;
28struct sock;
29struct seq_file;
30struct btf;
31struct btf_type;
32struct exception_table_entry;
33
34extern struct idr btf_idr;
35extern spinlock_t btf_idr_lock;
36
37
38struct bpf_map_ops {
39
40 int (*map_alloc_check)(union bpf_attr *attr);
41 struct bpf_map *(*map_alloc)(union bpf_attr *attr);
42 void (*map_release)(struct bpf_map *map, struct file *map_file);
43 void (*map_free)(struct bpf_map *map);
44 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
45 void (*map_release_uref)(struct bpf_map *map);
46 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
47 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,
48 union bpf_attr __user *uattr);
49 int (*map_lookup_and_delete_batch)(struct bpf_map *map,
50 const union bpf_attr *attr,
51 union bpf_attr __user *uattr);
52 int (*map_update_batch)(struct bpf_map *map, const union bpf_attr *attr,
53 union bpf_attr __user *uattr);
54 int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,
55 union bpf_attr __user *uattr);
56
57
58 void *(*map_lookup_elem)(struct bpf_map *map, void *key);
59 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
60 int (*map_delete_elem)(struct bpf_map *map, void *key);
61 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
62 int (*map_pop_elem)(struct bpf_map *map, void *value);
63 int (*map_peek_elem)(struct bpf_map *map, void *value);
64
65
66 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
67 int fd);
68 void (*map_fd_put_ptr)(void *ptr);
69 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
70 u32 (*map_fd_sys_lookup_elem)(void *ptr);
71 void (*map_seq_show_elem)(struct bpf_map *map, void *key,
72 struct seq_file *m);
73 int (*map_check_btf)(const struct bpf_map *map,
74 const struct btf *btf,
75 const struct btf_type *key_type,
76 const struct btf_type *value_type);
77
78
79 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux);
80 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux);
81 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old,
82 struct bpf_prog *new);
83
84
85 int (*map_direct_value_addr)(const struct bpf_map *map,
86 u64 *imm, u32 off);
87 int (*map_direct_value_meta)(const struct bpf_map *map,
88 u64 imm, u32 *off);
89 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
90};
91
92struct bpf_map_memory {
93 u32 pages;
94 struct user_struct *user;
95};
96
97struct bpf_map {
98
99
100
101 const struct bpf_map_ops *ops ____cacheline_aligned;
102 struct bpf_map *inner_map_meta;
103#ifdef CONFIG_SECURITY
104 void *security;
105#endif
106 enum bpf_map_type map_type;
107 u32 key_size;
108 u32 value_size;
109 u32 max_entries;
110 u32 map_flags;
111 int spin_lock_off;
112 u32 id;
113 int numa_node;
114 u32 btf_key_type_id;
115 u32 btf_value_type_id;
116 struct btf *btf;
117 struct bpf_map_memory memory;
118 char name[BPF_OBJ_NAME_LEN];
119 u32 btf_vmlinux_value_type_id;
120 bool unpriv_array;
121 bool frozen;
122
123
124
125
126
127 atomic64_t refcnt ____cacheline_aligned;
128 atomic64_t usercnt;
129 struct work_struct work;
130 struct mutex freeze_mutex;
131 u64 writecnt;
132};
133
134static inline bool map_value_has_spin_lock(const struct bpf_map *map)
135{
136 return map->spin_lock_off >= 0;
137}
138
139static inline void check_and_init_map_lock(struct bpf_map *map, void *dst)
140{
141 if (likely(!map_value_has_spin_lock(map)))
142 return;
143 *(struct bpf_spin_lock *)(dst + map->spin_lock_off) =
144 (struct bpf_spin_lock){};
145}
146
147
148static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
149{
150 if (unlikely(map_value_has_spin_lock(map))) {
151 u32 off = map->spin_lock_off;
152
153 memcpy(dst, src, off);
154 memcpy(dst + off + sizeof(struct bpf_spin_lock),
155 src + off + sizeof(struct bpf_spin_lock),
156 map->value_size - off - sizeof(struct bpf_spin_lock));
157 } else {
158 memcpy(dst, src, map->value_size);
159 }
160}
161void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
162 bool lock_src);
163int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);
164
165struct bpf_offload_dev;
166struct bpf_offloaded_map;
167
168struct bpf_map_dev_ops {
169 int (*map_get_next_key)(struct bpf_offloaded_map *map,
170 void *key, void *next_key);
171 int (*map_lookup_elem)(struct bpf_offloaded_map *map,
172 void *key, void *value);
173 int (*map_update_elem)(struct bpf_offloaded_map *map,
174 void *key, void *value, u64 flags);
175 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);
176};
177
178struct bpf_offloaded_map {
179 struct bpf_map map;
180 struct net_device *netdev;
181 const struct bpf_map_dev_ops *dev_ops;
182 void *dev_priv;
183 struct list_head offloads;
184};
185
186static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)
187{
188 return container_of(map, struct bpf_offloaded_map, map);
189}
190
191static inline bool bpf_map_offload_neutral(const struct bpf_map *map)
192{
193 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
194}
195
196static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
197{
198 return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) &&
199 map->ops->map_seq_show_elem;
200}
201
202int map_check_no_btf(const struct bpf_map *map,
203 const struct btf *btf,
204 const struct btf_type *key_type,
205 const struct btf_type *value_type);
206
207extern const struct bpf_map_ops bpf_map_offload_ops;
208
209
210enum bpf_arg_type {
211 ARG_DONTCARE = 0,
212
213
214
215
216 ARG_CONST_MAP_PTR,
217 ARG_PTR_TO_MAP_KEY,
218 ARG_PTR_TO_MAP_VALUE,
219 ARG_PTR_TO_UNINIT_MAP_VALUE,
220 ARG_PTR_TO_MAP_VALUE_OR_NULL,
221
222
223
224
225 ARG_PTR_TO_MEM,
226 ARG_PTR_TO_MEM_OR_NULL,
227 ARG_PTR_TO_UNINIT_MEM,
228
229
230
231
232 ARG_CONST_SIZE,
233 ARG_CONST_SIZE_OR_ZERO,
234
235 ARG_PTR_TO_CTX,
236 ARG_ANYTHING,
237 ARG_PTR_TO_SPIN_LOCK,
238 ARG_PTR_TO_SOCK_COMMON,
239 ARG_PTR_TO_INT,
240 ARG_PTR_TO_LONG,
241 ARG_PTR_TO_SOCKET,
242 ARG_PTR_TO_BTF_ID,
243};
244
245
246enum bpf_return_type {
247 RET_INTEGER,
248 RET_VOID,
249 RET_PTR_TO_MAP_VALUE,
250 RET_PTR_TO_MAP_VALUE_OR_NULL,
251 RET_PTR_TO_SOCKET_OR_NULL,
252 RET_PTR_TO_TCP_SOCK_OR_NULL,
253 RET_PTR_TO_SOCK_COMMON_OR_NULL,
254};
255
256
257
258
259
260struct bpf_func_proto {
261 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
262 bool gpl_only;
263 bool pkt_access;
264 enum bpf_return_type ret_type;
265 union {
266 struct {
267 enum bpf_arg_type arg1_type;
268 enum bpf_arg_type arg2_type;
269 enum bpf_arg_type arg3_type;
270 enum bpf_arg_type arg4_type;
271 enum bpf_arg_type arg5_type;
272 };
273 enum bpf_arg_type arg_type[5];
274 };
275 int *btf_id;
276};
277
278
279
280
281
282struct bpf_context;
283
284enum bpf_access_type {
285 BPF_READ = 1,
286 BPF_WRITE = 2
287};
288
289
290
291
292
293
294
295
296
297
298
299enum bpf_reg_type {
300 NOT_INIT = 0,
301 SCALAR_VALUE,
302 PTR_TO_CTX,
303 CONST_PTR_TO_MAP,
304 PTR_TO_MAP_VALUE,
305 PTR_TO_MAP_VALUE_OR_NULL,
306 PTR_TO_STACK,
307 PTR_TO_PACKET_META,
308 PTR_TO_PACKET,
309 PTR_TO_PACKET_END,
310 PTR_TO_FLOW_KEYS,
311 PTR_TO_SOCKET,
312 PTR_TO_SOCKET_OR_NULL,
313 PTR_TO_SOCK_COMMON,
314 PTR_TO_SOCK_COMMON_OR_NULL,
315 PTR_TO_TCP_SOCK,
316 PTR_TO_TCP_SOCK_OR_NULL,
317 PTR_TO_TP_BUFFER,
318 PTR_TO_XDP_SOCK,
319 PTR_TO_BTF_ID,
320};
321
322
323
324
325struct bpf_insn_access_aux {
326 enum bpf_reg_type reg_type;
327 union {
328 int ctx_field_size;
329 u32 btf_id;
330 };
331 struct bpf_verifier_log *log;
332};
333
334static inline void
335bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
336{
337 aux->ctx_field_size = size;
338}
339
340struct bpf_prog_ops {
341 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
342 union bpf_attr __user *uattr);
343};
344
345struct bpf_verifier_ops {
346
347 const struct bpf_func_proto *
348 (*get_func_proto)(enum bpf_func_id func_id,
349 const struct bpf_prog *prog);
350
351
352
353
354 bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
355 const struct bpf_prog *prog,
356 struct bpf_insn_access_aux *info);
357 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
358 const struct bpf_prog *prog);
359 int (*gen_ld_abs)(const struct bpf_insn *orig,
360 struct bpf_insn *insn_buf);
361 u32 (*convert_ctx_access)(enum bpf_access_type type,
362 const struct bpf_insn *src,
363 struct bpf_insn *dst,
364 struct bpf_prog *prog, u32 *target_size);
365 int (*btf_struct_access)(struct bpf_verifier_log *log,
366 const struct btf_type *t, int off, int size,
367 enum bpf_access_type atype,
368 u32 *next_btf_id);
369};
370
371struct bpf_prog_offload_ops {
372
373 int (*insn_hook)(struct bpf_verifier_env *env,
374 int insn_idx, int prev_insn_idx);
375 int (*finalize)(struct bpf_verifier_env *env);
376
377 int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
378 struct bpf_insn *insn);
379 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
380
381 int (*prepare)(struct bpf_prog *prog);
382 int (*translate)(struct bpf_prog *prog);
383 void (*destroy)(struct bpf_prog *prog);
384};
385
386struct bpf_prog_offload {
387 struct bpf_prog *prog;
388 struct net_device *netdev;
389 struct bpf_offload_dev *offdev;
390 void *dev_priv;
391 struct list_head offloads;
392 bool dev_state;
393 bool opt_failed;
394 void *jited_image;
395 u32 jited_len;
396};
397
398enum bpf_cgroup_storage_type {
399 BPF_CGROUP_STORAGE_SHARED,
400 BPF_CGROUP_STORAGE_PERCPU,
401 __BPF_CGROUP_STORAGE_MAX
402};
403
404#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
405
406
407
408
409#define MAX_BPF_FUNC_ARGS 12
410
411struct bpf_prog_stats {
412 u64 cnt;
413 u64 nsecs;
414 struct u64_stats_sync syncp;
415} __aligned(2 * sizeof(u64));
416
417struct btf_func_model {
418 u8 ret_size;
419 u8 nr_args;
420 u8 arg_size[MAX_BPF_FUNC_ARGS];
421};
422
423
424
425
426
427#define BPF_TRAMP_F_RESTORE_REGS BIT(0)
428
429
430
431#define BPF_TRAMP_F_CALL_ORIG BIT(1)
432
433
434
435#define BPF_TRAMP_F_SKIP_FRAME BIT(2)
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457int arch_prepare_bpf_trampoline(void *image, void *image_end,
458 const struct btf_func_model *m, u32 flags,
459 struct bpf_prog **fentry_progs, int fentry_cnt,
460 struct bpf_prog **fexit_progs, int fexit_cnt,
461 void *orig_call);
462
463u64 notrace __bpf_prog_enter(void);
464void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);
465
466enum bpf_tramp_prog_type {
467 BPF_TRAMP_FENTRY,
468 BPF_TRAMP_FEXIT,
469 BPF_TRAMP_MAX,
470 BPF_TRAMP_REPLACE,
471};
472
473struct bpf_trampoline {
474
475 struct hlist_node hlist;
476
477 struct mutex mutex;
478 refcount_t refcnt;
479 u64 key;
480 struct {
481 struct btf_func_model model;
482 void *addr;
483 bool ftrace_managed;
484 } func;
485
486
487
488
489 struct bpf_prog *extension_prog;
490
491 struct hlist_head progs_hlist[BPF_TRAMP_MAX];
492
493 int progs_cnt[BPF_TRAMP_MAX];
494
495 void *image;
496 u64 selector;
497};
498
499#define BPF_DISPATCHER_MAX 48
500
501struct bpf_dispatcher_prog {
502 struct bpf_prog *prog;
503 refcount_t users;
504};
505
506struct bpf_dispatcher {
507
508 struct mutex mutex;
509 void *func;
510 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
511 int num_progs;
512 void *image;
513 u32 image_off;
514};
515
516static __always_inline unsigned int bpf_dispatcher_nopfunc(
517 const void *ctx,
518 const struct bpf_insn *insnsi,
519 unsigned int (*bpf_func)(const void *,
520 const struct bpf_insn *))
521{
522 return bpf_func(ctx, insnsi);
523}
524#ifdef CONFIG_BPF_JIT
525struct bpf_trampoline *bpf_trampoline_lookup(u64 key);
526int bpf_trampoline_link_prog(struct bpf_prog *prog);
527int bpf_trampoline_unlink_prog(struct bpf_prog *prog);
528void bpf_trampoline_put(struct bpf_trampoline *tr);
529#define BPF_DISPATCHER_INIT(name) { \
530 .mutex = __MUTEX_INITIALIZER(name.mutex), \
531 .func = &name##func, \
532 .progs = {}, \
533 .num_progs = 0, \
534 .image = NULL, \
535 .image_off = 0 \
536}
537
538#define DEFINE_BPF_DISPATCHER(name) \
539 noinline unsigned int name##func( \
540 const void *ctx, \
541 const struct bpf_insn *insnsi, \
542 unsigned int (*bpf_func)(const void *, \
543 const struct bpf_insn *)) \
544 { \
545 return bpf_func(ctx, insnsi); \
546 } \
547 EXPORT_SYMBOL(name##func); \
548 struct bpf_dispatcher name = BPF_DISPATCHER_INIT(name);
549#define DECLARE_BPF_DISPATCHER(name) \
550 unsigned int name##func( \
551 const void *ctx, \
552 const struct bpf_insn *insnsi, \
553 unsigned int (*bpf_func)(const void *, \
554 const struct bpf_insn *)); \
555 extern struct bpf_dispatcher name;
556#define BPF_DISPATCHER_FUNC(name) name##func
557#define BPF_DISPATCHER_PTR(name) (&name)
558void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
559 struct bpf_prog *to);
560struct bpf_image {
561 struct latch_tree_node tnode;
562 unsigned char data[];
563};
564#define BPF_IMAGE_SIZE (PAGE_SIZE - sizeof(struct bpf_image))
565bool is_bpf_image_address(unsigned long address);
566void *bpf_image_alloc(void);
567#else
568static inline struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
569{
570 return NULL;
571}
572static inline int bpf_trampoline_link_prog(struct bpf_prog *prog)
573{
574 return -ENOTSUPP;
575}
576static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog)
577{
578 return -ENOTSUPP;
579}
580static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
581#define DEFINE_BPF_DISPATCHER(name)
582#define DECLARE_BPF_DISPATCHER(name)
583#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nopfunc
584#define BPF_DISPATCHER_PTR(name) NULL
585static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d,
586 struct bpf_prog *from,
587 struct bpf_prog *to) {}
588static inline bool is_bpf_image_address(unsigned long address)
589{
590 return false;
591}
592#endif
593
594struct bpf_func_info_aux {
595 u16 linkage;
596 bool unreliable;
597};
598
599enum bpf_jit_poke_reason {
600 BPF_POKE_REASON_TAIL_CALL,
601};
602
603
604struct bpf_jit_poke_descriptor {
605 void *ip;
606 union {
607 struct {
608 struct bpf_map *map;
609 u32 key;
610 } tail_call;
611 };
612 bool ip_stable;
613 u8 adj_off;
614 u16 reason;
615};
616
617struct bpf_prog_aux {
618 atomic64_t refcnt;
619 u32 used_map_cnt;
620 u32 max_ctx_offset;
621 u32 max_pkt_offset;
622 u32 max_tp_access;
623 u32 stack_depth;
624 u32 id;
625 u32 func_cnt;
626 u32 func_idx;
627 u32 attach_btf_id;
628 struct bpf_prog *linked_prog;
629 bool verifier_zext;
630 bool offload_requested;
631 bool attach_btf_trace;
632 bool func_proto_unreliable;
633 enum bpf_tramp_prog_type trampoline_prog_type;
634 struct bpf_trampoline *trampoline;
635 struct hlist_node tramp_hlist;
636
637 const struct btf_type *attach_func_proto;
638
639 const char *attach_func_name;
640 struct bpf_prog **func;
641 void *jit_data;
642 struct bpf_jit_poke_descriptor *poke_tab;
643 u32 size_poke_tab;
644 struct latch_tree_node ksym_tnode;
645 struct list_head ksym_lnode;
646 const struct bpf_prog_ops *ops;
647 struct bpf_map **used_maps;
648 struct bpf_prog *prog;
649 struct user_struct *user;
650 u64 load_time;
651 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
652 char name[BPF_OBJ_NAME_LEN];
653#ifdef CONFIG_SECURITY
654 void *security;
655#endif
656 struct bpf_prog_offload *offload;
657 struct btf *btf;
658 struct bpf_func_info *func_info;
659 struct bpf_func_info_aux *func_info_aux;
660
661
662
663
664
665
666 struct bpf_line_info *linfo;
667
668
669
670
671
672
673
674 void **jited_linfo;
675 u32 func_info_cnt;
676 u32 nr_linfo;
677
678
679
680
681 u32 linfo_idx;
682 u32 num_exentries;
683 struct exception_table_entry *extable;
684 struct bpf_prog_stats __percpu *stats;
685 union {
686 struct work_struct work;
687 struct rcu_head rcu;
688 };
689};
690
691struct bpf_array_aux {
692
693
694
695
696
697 enum bpf_prog_type type;
698 bool jited;
699
700 struct list_head poke_progs;
701 struct bpf_map *map;
702 struct mutex poke_mutex;
703 struct work_struct work;
704};
705
706struct bpf_struct_ops_value;
707struct btf_type;
708struct btf_member;
709
710#define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
711struct bpf_struct_ops {
712 const struct bpf_verifier_ops *verifier_ops;
713 int (*init)(struct btf *btf);
714 int (*check_member)(const struct btf_type *t,
715 const struct btf_member *member);
716 int (*init_member)(const struct btf_type *t,
717 const struct btf_member *member,
718 void *kdata, const void *udata);
719 int (*reg)(void *kdata);
720 void (*unreg)(void *kdata);
721 const struct btf_type *type;
722 const struct btf_type *value_type;
723 const char *name;
724 struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
725 u32 type_id;
726 u32 value_id;
727};
728
729#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
730#define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
731const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id);
732void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log);
733bool bpf_struct_ops_get(const void *kdata);
734void bpf_struct_ops_put(const void *kdata);
735int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
736 void *value);
737static inline bool bpf_try_module_get(const void *data, struct module *owner)
738{
739 if (owner == BPF_MODULE_OWNER)
740 return bpf_struct_ops_get(data);
741 else
742 return try_module_get(owner);
743}
744static inline void bpf_module_put(const void *data, struct module *owner)
745{
746 if (owner == BPF_MODULE_OWNER)
747 bpf_struct_ops_put(data);
748 else
749 module_put(owner);
750}
751#else
752static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
753{
754 return NULL;
755}
756static inline void bpf_struct_ops_init(struct btf *btf,
757 struct bpf_verifier_log *log)
758{
759}
760static inline bool bpf_try_module_get(const void *data, struct module *owner)
761{
762 return try_module_get(owner);
763}
764static inline void bpf_module_put(const void *data, struct module *owner)
765{
766 module_put(owner);
767}
768static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
769 void *key,
770 void *value)
771{
772 return -EINVAL;
773}
774#endif
775
776struct bpf_array {
777 struct bpf_map map;
778 u32 elem_size;
779 u32 index_mask;
780 struct bpf_array_aux *aux;
781 union {
782 char value[0] __aligned(8);
783 void *ptrs[0] __aligned(8);
784 void __percpu *pptrs[0] __aligned(8);
785 };
786};
787
788#define BPF_COMPLEXITY_LIMIT_INSNS 1000000
789#define MAX_TAIL_CALL_CNT 32
790
791#define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \
792 BPF_F_RDONLY_PROG | \
793 BPF_F_WRONLY | \
794 BPF_F_WRONLY_PROG)
795
796#define BPF_MAP_CAN_READ BIT(0)
797#define BPF_MAP_CAN_WRITE BIT(1)
798
799static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
800{
801 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
802
803
804
805
806 if (access_flags & BPF_F_RDONLY_PROG)
807 return BPF_MAP_CAN_READ;
808 else if (access_flags & BPF_F_WRONLY_PROG)
809 return BPF_MAP_CAN_WRITE;
810 else
811 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE;
812}
813
814static inline bool bpf_map_flags_access_ok(u32 access_flags)
815{
816 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=
817 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
818}
819
820struct bpf_event_entry {
821 struct perf_event *event;
822 struct file *perf_file;
823 struct file *map_file;
824 struct rcu_head rcu;
825};
826
827bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
828int bpf_prog_calc_tag(struct bpf_prog *fp);
829const char *kernel_type_name(u32 btf_type_id);
830
831const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
832
833typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
834 unsigned long off, unsigned long len);
835typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
836 const struct bpf_insn *src,
837 struct bpf_insn *dst,
838 struct bpf_prog *prog,
839 u32 *target_size);
840
841u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
842 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
843
844
845
846
847
848
849
850
851
852
853
854
855
856struct bpf_prog_array_item {
857 struct bpf_prog *prog;
858 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
859};
860
861struct bpf_prog_array {
862 struct rcu_head rcu;
863 struct bpf_prog_array_item items[0];
864};
865
866struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
867void bpf_prog_array_free(struct bpf_prog_array *progs);
868int bpf_prog_array_length(struct bpf_prog_array *progs);
869bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
870int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
871 __u32 __user *prog_ids, u32 cnt);
872
873void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
874 struct bpf_prog *old_prog);
875int bpf_prog_array_copy_info(struct bpf_prog_array *array,
876 u32 *prog_ids, u32 request_cnt,
877 u32 *prog_cnt);
878int bpf_prog_array_copy(struct bpf_prog_array *old_array,
879 struct bpf_prog *exclude_prog,
880 struct bpf_prog *include_prog,
881 struct bpf_prog_array **new_array);
882
883#define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \
884 ({ \
885 struct bpf_prog_array_item *_item; \
886 struct bpf_prog *_prog; \
887 struct bpf_prog_array *_array; \
888 u32 _ret = 1; \
889 preempt_disable(); \
890 rcu_read_lock(); \
891 _array = rcu_dereference(array); \
892 if (unlikely(check_non_null && !_array))\
893 goto _out; \
894 _item = &_array->items[0]; \
895 while ((_prog = READ_ONCE(_item->prog))) { \
896 bpf_cgroup_storage_set(_item->cgroup_storage); \
897 _ret &= func(_prog, ctx); \
898 _item++; \
899 } \
900_out: \
901 rcu_read_unlock(); \
902 preempt_enable(); \
903 _ret; \
904 })
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928#define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \
929 ({ \
930 struct bpf_prog_array_item *_item; \
931 struct bpf_prog *_prog; \
932 struct bpf_prog_array *_array; \
933 u32 ret; \
934 u32 _ret = 1; \
935 u32 _cn = 0; \
936 preempt_disable(); \
937 rcu_read_lock(); \
938 _array = rcu_dereference(array); \
939 _item = &_array->items[0]; \
940 while ((_prog = READ_ONCE(_item->prog))) { \
941 bpf_cgroup_storage_set(_item->cgroup_storage); \
942 ret = func(_prog, ctx); \
943 _ret &= (ret & 1); \
944 _cn |= (ret & 2); \
945 _item++; \
946 } \
947 rcu_read_unlock(); \
948 preempt_enable(); \
949 if (_ret) \
950 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \
951 else \
952 _ret = (_cn ? NET_XMIT_DROP : -EPERM); \
953 _ret; \
954 })
955
956#define BPF_PROG_RUN_ARRAY(array, ctx, func) \
957 __BPF_PROG_RUN_ARRAY(array, ctx, func, false)
958
959#define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \
960 __BPF_PROG_RUN_ARRAY(array, ctx, func, true)
961
962#ifdef CONFIG_BPF_SYSCALL
963DECLARE_PER_CPU(int, bpf_prog_active);
964
965extern const struct file_operations bpf_map_fops;
966extern const struct file_operations bpf_prog_fops;
967
968#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
969 extern const struct bpf_prog_ops _name ## _prog_ops; \
970 extern const struct bpf_verifier_ops _name ## _verifier_ops;
971#define BPF_MAP_TYPE(_id, _ops) \
972 extern const struct bpf_map_ops _ops;
973#include <linux/bpf_types.h>
974#undef BPF_PROG_TYPE
975#undef BPF_MAP_TYPE
976
977extern const struct bpf_prog_ops bpf_offload_prog_ops;
978extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
979extern const struct bpf_verifier_ops xdp_analyzer_ops;
980
981struct bpf_prog *bpf_prog_get(u32 ufd);
982struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
983 bool attach_drv);
984void bpf_prog_add(struct bpf_prog *prog, int i);
985void bpf_prog_sub(struct bpf_prog *prog, int i);
986void bpf_prog_inc(struct bpf_prog *prog);
987struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
988void bpf_prog_put(struct bpf_prog *prog);
989int __bpf_prog_charge(struct user_struct *user, u32 pages);
990void __bpf_prog_uncharge(struct user_struct *user, u32 pages);
991void __bpf_free_used_maps(struct bpf_prog_aux *aux,
992 struct bpf_map **used_maps, u32 len);
993
994void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock);
995void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock);
996
997struct bpf_map *bpf_map_get_with_uref(u32 ufd);
998struct bpf_map *__bpf_map_get(struct fd f);
999void bpf_map_inc(struct bpf_map *map);
1000void bpf_map_inc_with_uref(struct bpf_map *map);
1001struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);
1002void bpf_map_put_with_uref(struct bpf_map *map);
1003void bpf_map_put(struct bpf_map *map);
1004int bpf_map_charge_memlock(struct bpf_map *map, u32 pages);
1005void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages);
1006int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size);
1007void bpf_map_charge_finish(struct bpf_map_memory *mem);
1008void bpf_map_charge_move(struct bpf_map_memory *dst,
1009 struct bpf_map_memory *src);
1010void *bpf_map_area_alloc(u64 size, int numa_node);
1011void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
1012void bpf_map_area_free(void *base);
1013void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
1014int generic_map_lookup_batch(struct bpf_map *map,
1015 const union bpf_attr *attr,
1016 union bpf_attr __user *uattr);
1017int generic_map_update_batch(struct bpf_map *map,
1018 const union bpf_attr *attr,
1019 union bpf_attr __user *uattr);
1020int generic_map_delete_batch(struct bpf_map *map,
1021 const union bpf_attr *attr,
1022 union bpf_attr __user *uattr);
1023
1024extern int sysctl_unprivileged_bpf_disabled;
1025
1026int bpf_map_new_fd(struct bpf_map *map, int flags);
1027int bpf_prog_new_fd(struct bpf_prog *prog);
1028
1029int bpf_obj_pin_user(u32 ufd, const char __user *pathname);
1030int bpf_obj_get_user(const char __user *pathname, int flags);
1031
1032int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
1033int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
1034int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
1035 u64 flags);
1036int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
1037 u64 flags);
1038
1039int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
1040
1041int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
1042 void *key, void *value, u64 map_flags);
1043int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
1044int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
1045 void *key, void *value, u64 map_flags);
1046int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
1047
1048int bpf_get_file_flag(int flags);
1049int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size,
1050 size_t actual_size);
1051
1052
1053
1054
1055
1056
1057
1058static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
1059{
1060 const long *lsrc = src;
1061 long *ldst = dst;
1062
1063 size /= sizeof(long);
1064 while (size--)
1065 *ldst++ = *lsrc++;
1066}
1067
1068
1069int bpf_check(struct bpf_prog **fp, union bpf_attr *attr,
1070 union bpf_attr __user *uattr);
1071void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
1072
1073
1074struct xdp_buff;
1075struct sk_buff;
1076
1077struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key);
1078struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key);
1079void __dev_flush(void);
1080int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,
1081 struct net_device *dev_rx);
1082int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,
1083 struct net_device *dev_rx);
1084int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
1085 struct bpf_prog *xdp_prog);
1086
1087struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key);
1088void __cpu_map_flush(void);
1089int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
1090 struct net_device *dev_rx);
1091
1092
1093static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
1094{
1095 return (attr->map_flags & BPF_F_NUMA_NODE) ?
1096 attr->numa_node : NUMA_NO_NODE;
1097}
1098
1099struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type);
1100int array_map_alloc_check(union bpf_attr *attr);
1101
1102int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
1103 union bpf_attr __user *uattr);
1104int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
1105 union bpf_attr __user *uattr);
1106int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
1107 const union bpf_attr *kattr,
1108 union bpf_attr __user *uattr);
1109bool btf_ctx_access(int off, int size, enum bpf_access_type type,
1110 const struct bpf_prog *prog,
1111 struct bpf_insn_access_aux *info);
1112int btf_struct_access(struct bpf_verifier_log *log,
1113 const struct btf_type *t, int off, int size,
1114 enum bpf_access_type atype,
1115 u32 *next_btf_id);
1116int btf_resolve_helper_id(struct bpf_verifier_log *log,
1117 const struct bpf_func_proto *fn, int);
1118
1119int btf_distill_func_proto(struct bpf_verifier_log *log,
1120 struct btf *btf,
1121 const struct btf_type *func_proto,
1122 const char *func_name,
1123 struct btf_func_model *m);
1124
1125struct bpf_reg_state;
1126int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
1127 struct bpf_reg_state *regs);
1128int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
1129 struct bpf_reg_state *reg);
1130int btf_check_type_match(struct bpf_verifier_env *env, struct bpf_prog *prog,
1131 struct btf *btf, const struct btf_type *t);
1132
1133struct bpf_prog *bpf_prog_by_id(u32 id);
1134
1135#else
1136static inline struct bpf_prog *bpf_prog_get(u32 ufd)
1137{
1138 return ERR_PTR(-EOPNOTSUPP);
1139}
1140
1141static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
1142 enum bpf_prog_type type,
1143 bool attach_drv)
1144{
1145 return ERR_PTR(-EOPNOTSUPP);
1146}
1147
1148static inline void bpf_prog_add(struct bpf_prog *prog, int i)
1149{
1150}
1151
1152static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
1153{
1154}
1155
1156static inline void bpf_prog_put(struct bpf_prog *prog)
1157{
1158}
1159
1160static inline void bpf_prog_inc(struct bpf_prog *prog)
1161{
1162}
1163
1164static inline struct bpf_prog *__must_check
1165bpf_prog_inc_not_zero(struct bpf_prog *prog)
1166{
1167 return ERR_PTR(-EOPNOTSUPP);
1168}
1169
1170static inline int __bpf_prog_charge(struct user_struct *user, u32 pages)
1171{
1172 return 0;
1173}
1174
1175static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
1176{
1177}
1178
1179static inline int bpf_obj_get_user(const char __user *pathname, int flags)
1180{
1181 return -EOPNOTSUPP;
1182}
1183
1184static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map,
1185 u32 key)
1186{
1187 return NULL;
1188}
1189
1190static inline struct net_device *__dev_map_hash_lookup_elem(struct bpf_map *map,
1191 u32 key)
1192{
1193 return NULL;
1194}
1195
1196static inline void __dev_flush(void)
1197{
1198}
1199
1200struct xdp_buff;
1201struct bpf_dtab_netdev;
1202
1203static inline
1204int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,
1205 struct net_device *dev_rx)
1206{
1207 return 0;
1208}
1209
1210static inline
1211int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,
1212 struct net_device *dev_rx)
1213{
1214 return 0;
1215}
1216
1217struct sk_buff;
1218
1219static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
1220 struct sk_buff *skb,
1221 struct bpf_prog *xdp_prog)
1222{
1223 return 0;
1224}
1225
1226static inline
1227struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
1228{
1229 return NULL;
1230}
1231
1232static inline void __cpu_map_flush(void)
1233{
1234}
1235
1236static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
1237 struct xdp_buff *xdp,
1238 struct net_device *dev_rx)
1239{
1240 return 0;
1241}
1242
1243static inline struct bpf_prog *bpf_prog_get_type_path(const char *name,
1244 enum bpf_prog_type type)
1245{
1246 return ERR_PTR(-EOPNOTSUPP);
1247}
1248
1249static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,
1250 const union bpf_attr *kattr,
1251 union bpf_attr __user *uattr)
1252{
1253 return -ENOTSUPP;
1254}
1255
1256static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,
1257 const union bpf_attr *kattr,
1258 union bpf_attr __user *uattr)
1259{
1260 return -ENOTSUPP;
1261}
1262
1263static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
1264 const union bpf_attr *kattr,
1265 union bpf_attr __user *uattr)
1266{
1267 return -ENOTSUPP;
1268}
1269
1270static inline void bpf_map_put(struct bpf_map *map)
1271{
1272}
1273
1274static inline struct bpf_prog *bpf_prog_by_id(u32 id)
1275{
1276 return ERR_PTR(-ENOTSUPP);
1277}
1278#endif
1279
1280static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
1281 enum bpf_prog_type type)
1282{
1283 return bpf_prog_get_type_dev(ufd, type, false);
1284}
1285
1286bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);
1287
1288int bpf_prog_offload_compile(struct bpf_prog *prog);
1289void bpf_prog_offload_destroy(struct bpf_prog *prog);
1290int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
1291 struct bpf_prog *prog);
1292
1293int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map);
1294
1295int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value);
1296int bpf_map_offload_update_elem(struct bpf_map *map,
1297 void *key, void *value, u64 flags);
1298int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
1299int bpf_map_offload_get_next_key(struct bpf_map *map,
1300 void *key, void *next_key);
1301
1302bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);
1303
1304struct bpf_offload_dev *
1305bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv);
1306void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
1307void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev);
1308int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
1309 struct net_device *netdev);
1310void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
1311 struct net_device *netdev);
1312bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
1313
1314#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
1315int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr);
1316
1317static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
1318{
1319 return aux->offload_requested;
1320}
1321
1322static inline bool bpf_map_is_dev_bound(struct bpf_map *map)
1323{
1324 return unlikely(map->ops == &bpf_map_offload_ops);
1325}
1326
1327struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);
1328void bpf_map_offload_map_free(struct bpf_map *map);
1329#else
1330static inline int bpf_prog_offload_init(struct bpf_prog *prog,
1331 union bpf_attr *attr)
1332{
1333 return -EOPNOTSUPP;
1334}
1335
1336static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux)
1337{
1338 return false;
1339}
1340
1341static inline bool bpf_map_is_dev_bound(struct bpf_map *map)
1342{
1343 return false;
1344}
1345
1346static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
1347{
1348 return ERR_PTR(-EOPNOTSUPP);
1349}
1350
1351static inline void bpf_map_offload_map_free(struct bpf_map *map)
1352{
1353}
1354#endif
1355
1356#if defined(CONFIG_BPF_STREAM_PARSER)
1357int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, u32 which);
1358int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog);
1359#else
1360static inline int sock_map_prog_update(struct bpf_map *map,
1361 struct bpf_prog *prog, u32 which)
1362{
1363 return -EOPNOTSUPP;
1364}
1365
1366static inline int sock_map_get_from_fd(const union bpf_attr *attr,
1367 struct bpf_prog *prog)
1368{
1369 return -EINVAL;
1370}
1371#endif
1372
1373#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
1374void bpf_sk_reuseport_detach(struct sock *sk);
1375int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
1376 void *value);
1377int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
1378 void *value, u64 map_flags);
1379#else
1380static inline void bpf_sk_reuseport_detach(struct sock *sk)
1381{
1382}
1383
1384#ifdef CONFIG_BPF_SYSCALL
1385static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,
1386 void *key, void *value)
1387{
1388 return -EOPNOTSUPP;
1389}
1390
1391static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
1392 void *key, void *value,
1393 u64 map_flags)
1394{
1395 return -EOPNOTSUPP;
1396}
1397#endif
1398#endif
1399
1400
1401extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
1402extern const struct bpf_func_proto bpf_map_update_elem_proto;
1403extern const struct bpf_func_proto bpf_map_delete_elem_proto;
1404extern const struct bpf_func_proto bpf_map_push_elem_proto;
1405extern const struct bpf_func_proto bpf_map_pop_elem_proto;
1406extern const struct bpf_func_proto bpf_map_peek_elem_proto;
1407
1408extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
1409extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
1410extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
1411extern const struct bpf_func_proto bpf_tail_call_proto;
1412extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
1413extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
1414extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
1415extern const struct bpf_func_proto bpf_get_current_comm_proto;
1416extern const struct bpf_func_proto bpf_get_stackid_proto;
1417extern const struct bpf_func_proto bpf_get_stack_proto;
1418extern const struct bpf_func_proto bpf_sock_map_update_proto;
1419extern const struct bpf_func_proto bpf_sock_hash_update_proto;
1420extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
1421extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
1422extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
1423extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
1424extern const struct bpf_func_proto bpf_sk_redirect_map_proto;
1425extern const struct bpf_func_proto bpf_spin_lock_proto;
1426extern const struct bpf_func_proto bpf_spin_unlock_proto;
1427extern const struct bpf_func_proto bpf_get_local_storage_proto;
1428extern const struct bpf_func_proto bpf_strtol_proto;
1429extern const struct bpf_func_proto bpf_strtoul_proto;
1430extern const struct bpf_func_proto bpf_tcp_sock_proto;
1431extern const struct bpf_func_proto bpf_jiffies64_proto;
1432
1433
1434void bpf_user_rnd_init_once(void);
1435u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
1436
1437#if defined(CONFIG_NET)
1438bool bpf_sock_common_is_valid_access(int off, int size,
1439 enum bpf_access_type type,
1440 struct bpf_insn_access_aux *info);
1441bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
1442 struct bpf_insn_access_aux *info);
1443u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
1444 const struct bpf_insn *si,
1445 struct bpf_insn *insn_buf,
1446 struct bpf_prog *prog,
1447 u32 *target_size);
1448#else
1449static inline bool bpf_sock_common_is_valid_access(int off, int size,
1450 enum bpf_access_type type,
1451 struct bpf_insn_access_aux *info)
1452{
1453 return false;
1454}
1455static inline bool bpf_sock_is_valid_access(int off, int size,
1456 enum bpf_access_type type,
1457 struct bpf_insn_access_aux *info)
1458{
1459 return false;
1460}
1461static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
1462 const struct bpf_insn *si,
1463 struct bpf_insn *insn_buf,
1464 struct bpf_prog *prog,
1465 u32 *target_size)
1466{
1467 return 0;
1468}
1469#endif
1470
1471#ifdef CONFIG_INET
1472struct sk_reuseport_kern {
1473 struct sk_buff *skb;
1474 struct sock *sk;
1475 struct sock *selected_sk;
1476 void *data_end;
1477 u32 hash;
1478 u32 reuseport_id;
1479 bool bind_inany;
1480};
1481bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
1482 struct bpf_insn_access_aux *info);
1483
1484u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
1485 const struct bpf_insn *si,
1486 struct bpf_insn *insn_buf,
1487 struct bpf_prog *prog,
1488 u32 *target_size);
1489
1490bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
1491 struct bpf_insn_access_aux *info);
1492
1493u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
1494 const struct bpf_insn *si,
1495 struct bpf_insn *insn_buf,
1496 struct bpf_prog *prog,
1497 u32 *target_size);
1498#else
1499static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
1500 enum bpf_access_type type,
1501 struct bpf_insn_access_aux *info)
1502{
1503 return false;
1504}
1505
1506static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
1507 const struct bpf_insn *si,
1508 struct bpf_insn *insn_buf,
1509 struct bpf_prog *prog,
1510 u32 *target_size)
1511{
1512 return 0;
1513}
1514static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
1515 enum bpf_access_type type,
1516 struct bpf_insn_access_aux *info)
1517{
1518 return false;
1519}
1520
1521static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
1522 const struct bpf_insn *si,
1523 struct bpf_insn *insn_buf,
1524 struct bpf_prog *prog,
1525 u32 *target_size)
1526{
1527 return 0;
1528}
1529#endif
1530
1531enum bpf_text_poke_type {
1532 BPF_MOD_CALL,
1533 BPF_MOD_JUMP,
1534};
1535
1536int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
1537 void *addr1, void *addr2);
1538
1539#endif
1540