1
2
3
4#ifndef _LINUX_BPF_H
5#define _LINUX_BPF_H 1
6
7#include <uapi/linux/bpf.h>
8
9#include <linux/workqueue.h>
10#include <linux/file.h>
11#include <linux/percpu.h>
12#include <linux/err.h>
13#include <linux/rbtree_latch.h>
14#include <linux/numa.h>
15#include <linux/wait.h>
16#include <linux/u64_stats_sync.h>
17
18struct bpf_verifier_env;
19struct perf_event;
20struct bpf_prog;
21struct bpf_map;
22struct sock;
23struct seq_file;
24struct btf;
25struct btf_type;
26
27
28struct bpf_map_ops {
29
30 int (*map_alloc_check)(union bpf_attr *attr);
31 struct bpf_map *(*map_alloc)(union bpf_attr *attr);
32 void (*map_release)(struct bpf_map *map, struct file *map_file);
33 void (*map_free)(struct bpf_map *map);
34 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
35 void (*map_release_uref)(struct bpf_map *map);
36 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
37
38
39 void *(*map_lookup_elem)(struct bpf_map *map, void *key);
40 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
41 int (*map_delete_elem)(struct bpf_map *map, void *key);
42 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
43 int (*map_pop_elem)(struct bpf_map *map, void *value);
44 int (*map_peek_elem)(struct bpf_map *map, void *value);
45
46
47 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
48 int fd);
49 void (*map_fd_put_ptr)(void *ptr);
50 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
51 u32 (*map_fd_sys_lookup_elem)(void *ptr);
52 void (*map_seq_show_elem)(struct bpf_map *map, void *key,
53 struct seq_file *m);
54 int (*map_check_btf)(const struct bpf_map *map,
55 const struct btf *btf,
56 const struct btf_type *key_type,
57 const struct btf_type *value_type);
58
59
60 int (*map_direct_value_addr)(const struct bpf_map *map,
61 u64 *imm, u32 off);
62 int (*map_direct_value_meta)(const struct bpf_map *map,
63 u64 imm, u32 *off);
64};
65
66struct bpf_map_memory {
67 u32 pages;
68 struct user_struct *user;
69};
70
71struct bpf_map {
72
73
74
75 const struct bpf_map_ops *ops ____cacheline_aligned;
76 struct bpf_map *inner_map_meta;
77#ifdef CONFIG_SECURITY
78 void *security;
79#endif
80 enum bpf_map_type map_type;
81 u32 key_size;
82 u32 value_size;
83 u32 max_entries;
84 u32 map_flags;
85 int spin_lock_off;
86 u32 id;
87 int numa_node;
88 u32 btf_key_type_id;
89 u32 btf_value_type_id;
90 struct btf *btf;
91 struct bpf_map_memory memory;
92 bool unpriv_array;
93 bool frozen;
94
95
96
97
98
99 atomic_t refcnt ____cacheline_aligned;
100 atomic_t usercnt;
101 struct work_struct work;
102 char name[BPF_OBJ_NAME_LEN];
103};
104
105static inline bool map_value_has_spin_lock(const struct bpf_map *map)
106{
107 return map->spin_lock_off >= 0;
108}
109
110static inline void check_and_init_map_lock(struct bpf_map *map, void *dst)
111{
112 if (likely(!map_value_has_spin_lock(map)))
113 return;
114 *(struct bpf_spin_lock *)(dst + map->spin_lock_off) =
115 (struct bpf_spin_lock){};
116}
117
118
119static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
120{
121 if (unlikely(map_value_has_spin_lock(map))) {
122 u32 off = map->spin_lock_off;
123
124 memcpy(dst, src, off);
125 memcpy(dst + off + sizeof(struct bpf_spin_lock),
126 src + off + sizeof(struct bpf_spin_lock),
127 map->value_size - off - sizeof(struct bpf_spin_lock));
128 } else {
129 memcpy(dst, src, map->value_size);
130 }
131}
132void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
133 bool lock_src);
134
135struct bpf_offload_dev;
136struct bpf_offloaded_map;
137
138struct bpf_map_dev_ops {
139 int (*map_get_next_key)(struct bpf_offloaded_map *map,
140 void *key, void *next_key);
141 int (*map_lookup_elem)(struct bpf_offloaded_map *map,
142 void *key, void *value);
143 int (*map_update_elem)(struct bpf_offloaded_map *map,
144 void *key, void *value, u64 flags);
145 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);
146};
147
148struct bpf_offloaded_map {
149 struct bpf_map map;
150 struct net_device *netdev;
151 const struct bpf_map_dev_ops *dev_ops;
152 void *dev_priv;
153 struct list_head offloads;
154};
155
156static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)
157{
158 return container_of(map, struct bpf_offloaded_map, map);
159}
160
161static inline bool bpf_map_offload_neutral(const struct bpf_map *map)
162{
163 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
164}
165
166static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
167{
168 return map->btf && map->ops->map_seq_show_elem;
169}
170
171int map_check_no_btf(const struct bpf_map *map,
172 const struct btf *btf,
173 const struct btf_type *key_type,
174 const struct btf_type *value_type);
175
176extern const struct bpf_map_ops bpf_map_offload_ops;
177
178
179enum bpf_arg_type {
180 ARG_DONTCARE = 0,
181
182
183
184
185 ARG_CONST_MAP_PTR,
186 ARG_PTR_TO_MAP_KEY,
187 ARG_PTR_TO_MAP_VALUE,
188 ARG_PTR_TO_UNINIT_MAP_VALUE,
189 ARG_PTR_TO_MAP_VALUE_OR_NULL,
190
191
192
193
194 ARG_PTR_TO_MEM,
195 ARG_PTR_TO_MEM_OR_NULL,
196 ARG_PTR_TO_UNINIT_MEM,
197
198
199
200
201 ARG_CONST_SIZE,
202 ARG_CONST_SIZE_OR_ZERO,
203
204 ARG_PTR_TO_CTX,
205 ARG_ANYTHING,
206 ARG_PTR_TO_SPIN_LOCK,
207 ARG_PTR_TO_SOCK_COMMON,
208 ARG_PTR_TO_INT,
209 ARG_PTR_TO_LONG,
210 ARG_PTR_TO_SOCKET,
211};
212
213
214enum bpf_return_type {
215 RET_INTEGER,
216 RET_VOID,
217 RET_PTR_TO_MAP_VALUE,
218 RET_PTR_TO_MAP_VALUE_OR_NULL,
219 RET_PTR_TO_SOCKET_OR_NULL,
220 RET_PTR_TO_TCP_SOCK_OR_NULL,
221 RET_PTR_TO_SOCK_COMMON_OR_NULL,
222};
223
224
225
226
227
228struct bpf_func_proto {
229 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
230 bool gpl_only;
231 bool pkt_access;
232 enum bpf_return_type ret_type;
233 enum bpf_arg_type arg1_type;
234 enum bpf_arg_type arg2_type;
235 enum bpf_arg_type arg3_type;
236 enum bpf_arg_type arg4_type;
237 enum bpf_arg_type arg5_type;
238};
239
240
241
242
243
244struct bpf_context;
245
246enum bpf_access_type {
247 BPF_READ = 1,
248 BPF_WRITE = 2
249};
250
251
252
253
254
255
256
257
258
259
260
261enum bpf_reg_type {
262 NOT_INIT = 0,
263 SCALAR_VALUE,
264 PTR_TO_CTX,
265 CONST_PTR_TO_MAP,
266 PTR_TO_MAP_VALUE,
267 PTR_TO_MAP_VALUE_OR_NULL,
268 PTR_TO_STACK,
269 PTR_TO_PACKET_META,
270 PTR_TO_PACKET,
271 PTR_TO_PACKET_END,
272 PTR_TO_FLOW_KEYS,
273 PTR_TO_SOCKET,
274 PTR_TO_SOCKET_OR_NULL,
275 PTR_TO_SOCK_COMMON,
276 PTR_TO_SOCK_COMMON_OR_NULL,
277 PTR_TO_TCP_SOCK,
278 PTR_TO_TCP_SOCK_OR_NULL,
279 PTR_TO_TP_BUFFER,
280 PTR_TO_XDP_SOCK,
281};
282
283
284
285
286struct bpf_insn_access_aux {
287 enum bpf_reg_type reg_type;
288 int ctx_field_size;
289};
290
291static inline void
292bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
293{
294 aux->ctx_field_size = size;
295}
296
297struct bpf_prog_ops {
298 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
299 union bpf_attr __user *uattr);
300};
301
302struct bpf_verifier_ops {
303
304 const struct bpf_func_proto *
305 (*get_func_proto)(enum bpf_func_id func_id,
306 const struct bpf_prog *prog);
307
308
309
310
311 bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
312 const struct bpf_prog *prog,
313 struct bpf_insn_access_aux *info);
314 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
315 const struct bpf_prog *prog);
316 int (*gen_ld_abs)(const struct bpf_insn *orig,
317 struct bpf_insn *insn_buf);
318 u32 (*convert_ctx_access)(enum bpf_access_type type,
319 const struct bpf_insn *src,
320 struct bpf_insn *dst,
321 struct bpf_prog *prog, u32 *target_size);
322};
323
324struct bpf_prog_offload_ops {
325
326 int (*insn_hook)(struct bpf_verifier_env *env,
327 int insn_idx, int prev_insn_idx);
328 int (*finalize)(struct bpf_verifier_env *env);
329
330 int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
331 struct bpf_insn *insn);
332 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
333
334 int (*prepare)(struct bpf_prog *prog);
335 int (*translate)(struct bpf_prog *prog);
336 void (*destroy)(struct bpf_prog *prog);
337};
338
339struct bpf_prog_offload {
340 struct bpf_prog *prog;
341 struct net_device *netdev;
342 struct bpf_offload_dev *offdev;
343 void *dev_priv;
344 struct list_head offloads;
345 bool dev_state;
346 bool opt_failed;
347 void *jited_image;
348 u32 jited_len;
349};
350
351enum bpf_cgroup_storage_type {
352 BPF_CGROUP_STORAGE_SHARED,
353 BPF_CGROUP_STORAGE_PERCPU,
354 __BPF_CGROUP_STORAGE_MAX
355};
356
357#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
358
359struct bpf_prog_stats {
360 u64 cnt;
361 u64 nsecs;
362 struct u64_stats_sync syncp;
363};
364
365struct bpf_prog_aux {
366 atomic_t refcnt;
367 u32 used_map_cnt;
368 u32 max_ctx_offset;
369 u32 max_pkt_offset;
370 u32 max_tp_access;
371 u32 stack_depth;
372 u32 id;
373 u32 func_cnt;
374 u32 func_idx;
375 bool verifier_zext;
376 bool offload_requested;
377 struct bpf_prog **func;
378 void *jit_data;
379 struct latch_tree_node ksym_tnode;
380 struct list_head ksym_lnode;
381 const struct bpf_prog_ops *ops;
382 struct bpf_map **used_maps;
383 struct bpf_prog *prog;
384 struct user_struct *user;
385 u64 load_time;
386 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
387 char name[BPF_OBJ_NAME_LEN];
388#ifdef CONFIG_SECURITY
389 void *security;
390#endif
391 struct bpf_prog_offload *offload;
392 struct btf *btf;
393 struct bpf_func_info *func_info;
394
395
396
397
398
399
400 struct bpf_line_info *linfo;
401
402
403
404
405
406
407
408 void **jited_linfo;
409 u32 func_info_cnt;
410 u32 nr_linfo;
411
412
413
414
415 u32 linfo_idx;
416 struct bpf_prog_stats __percpu *stats;
417 union {
418 struct work_struct work;
419 struct rcu_head rcu;
420 };
421};
422
423struct bpf_array {
424 struct bpf_map map;
425 u32 elem_size;
426 u32 index_mask;
427
428
429
430
431
432 enum bpf_prog_type owner_prog_type;
433 bool owner_jited;
434 union {
435 char value[0] __aligned(8);
436 void *ptrs[0] __aligned(8);
437 void __percpu *pptrs[0] __aligned(8);
438 };
439};
440
441#define BPF_COMPLEXITY_LIMIT_INSNS 1000000
442#define MAX_TAIL_CALL_CNT 32
443
444#define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \
445 BPF_F_RDONLY_PROG | \
446 BPF_F_WRONLY | \
447 BPF_F_WRONLY_PROG)
448
449#define BPF_MAP_CAN_READ BIT(0)
450#define BPF_MAP_CAN_WRITE BIT(1)
451
452static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
453{
454 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
455
456
457
458
459 if (access_flags & BPF_F_RDONLY_PROG)
460 return BPF_MAP_CAN_READ;
461 else if (access_flags & BPF_F_WRONLY_PROG)
462 return BPF_MAP_CAN_WRITE;
463 else
464 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE;
465}
466
467static inline bool bpf_map_flags_access_ok(u32 access_flags)
468{
469 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=
470 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
471}
472
473struct bpf_event_entry {
474 struct perf_event *event;
475 struct file *perf_file;
476 struct file *map_file;
477 struct rcu_head rcu;
478};
479
480bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
481int bpf_prog_calc_tag(struct bpf_prog *fp);
482
483const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
484
485typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
486 unsigned long off, unsigned long len);
487typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
488 const struct bpf_insn *src,
489 struct bpf_insn *dst,
490 struct bpf_prog *prog,
491 u32 *target_size);
492
493u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
494 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
495
496
497
498
499
500
501
502
503
504
505
506
507
508struct bpf_prog_array_item {
509 struct bpf_prog *prog;
510 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
511};
512
513struct bpf_prog_array {
514 struct rcu_head rcu;
515 struct bpf_prog_array_item items[0];
516};
517
518struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
519void bpf_prog_array_free(struct bpf_prog_array *progs);
520int bpf_prog_array_length(struct bpf_prog_array *progs);
521bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
522int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
523 __u32 __user *prog_ids, u32 cnt);
524
525void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
526 struct bpf_prog *old_prog);
527int bpf_prog_array_copy_info(struct bpf_prog_array *array,
528 u32 *prog_ids, u32 request_cnt,
529 u32 *prog_cnt);
530int bpf_prog_array_copy(struct bpf_prog_array *old_array,
531 struct bpf_prog *exclude_prog,
532 struct bpf_prog *include_prog,
533 struct bpf_prog_array **new_array);
534
535#define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \
536 ({ \
537 struct bpf_prog_array_item *_item; \
538 struct bpf_prog *_prog; \
539 struct bpf_prog_array *_array; \
540 u32 _ret = 1; \
541 preempt_disable(); \
542 rcu_read_lock(); \
543 _array = rcu_dereference(array); \
544 if (unlikely(check_non_null && !_array))\
545 goto _out; \
546 _item = &_array->items[0]; \
547 while ((_prog = READ_ONCE(_item->prog))) { \
548 bpf_cgroup_storage_set(_item->cgroup_storage); \
549 _ret &= func(_prog, ctx); \
550 _item++; \
551 } \
552_out: \
553 rcu_read_unlock(); \
554 preempt_enable(); \
555 _ret; \
556 })
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580#define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \
581 ({ \
582 struct bpf_prog_array_item *_item; \
583 struct bpf_prog *_prog; \
584 struct bpf_prog_array *_array; \
585 u32 ret; \
586 u32 _ret = 1; \
587 u32 _cn = 0; \
588 preempt_disable(); \
589 rcu_read_lock(); \
590 _array = rcu_dereference(array); \
591 _item = &_array->items[0]; \
592 while ((_prog = READ_ONCE(_item->prog))) { \
593 bpf_cgroup_storage_set(_item->cgroup_storage); \
594 ret = func(_prog, ctx); \
595 _ret &= (ret & 1); \
596 _cn |= (ret & 2); \
597 _item++; \
598 } \
599 rcu_read_unlock(); \
600 preempt_enable(); \
601 if (_ret) \
602 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \
603 else \
604 _ret = (_cn ? NET_XMIT_DROP : -EPERM); \
605 _ret; \
606 })
607
608#define BPF_PROG_RUN_ARRAY(array, ctx, func) \
609 __BPF_PROG_RUN_ARRAY(array, ctx, func, false)
610
611#define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \
612 __BPF_PROG_RUN_ARRAY(array, ctx, func, true)
613
614#ifdef CONFIG_BPF_SYSCALL
615DECLARE_PER_CPU(int, bpf_prog_active);
616
617extern const struct file_operations bpf_map_fops;
618extern const struct file_operations bpf_prog_fops;
619
620#define BPF_PROG_TYPE(_id, _name) \
621 extern const struct bpf_prog_ops _name ## _prog_ops; \
622 extern const struct bpf_verifier_ops _name ## _verifier_ops;
623#define BPF_MAP_TYPE(_id, _ops) \
624 extern const struct bpf_map_ops _ops;
625#include <linux/bpf_types.h>
626#undef BPF_PROG_TYPE
627#undef BPF_MAP_TYPE
628
629extern const struct bpf_prog_ops bpf_offload_prog_ops;
630extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
631extern const struct bpf_verifier_ops xdp_analyzer_ops;
632
633struct bpf_prog *bpf_prog_get(u32 ufd);
634struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
635 bool attach_drv);
636struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i);
637void bpf_prog_sub(struct bpf_prog *prog, int i);
638struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog);
639struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
640void bpf_prog_put(struct bpf_prog *prog);
641int __bpf_prog_charge(struct user_struct *user, u32 pages);
642void __bpf_prog_uncharge(struct user_struct *user, u32 pages);
643
644void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock);
645void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock);
646
647struct bpf_map *bpf_map_get_with_uref(u32 ufd);
648struct bpf_map *__bpf_map_get(struct fd f);
649struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref);
650void bpf_map_put_with_uref(struct bpf_map *map);
651void bpf_map_put(struct bpf_map *map);
652int bpf_map_charge_memlock(struct bpf_map *map, u32 pages);
653void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages);
654int bpf_map_charge_init(struct bpf_map_memory *mem, size_t size);
655void bpf_map_charge_finish(struct bpf_map_memory *mem);
656void bpf_map_charge_move(struct bpf_map_memory *dst,
657 struct bpf_map_memory *src);
658void *bpf_map_area_alloc(size_t size, int numa_node);
659void bpf_map_area_free(void *base);
660void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
661
662extern int sysctl_unprivileged_bpf_disabled;
663
664int bpf_map_new_fd(struct bpf_map *map, int flags);
665int bpf_prog_new_fd(struct bpf_prog *prog);
666
667int bpf_obj_pin_user(u32 ufd, const char __user *pathname);
668int bpf_obj_get_user(const char __user *pathname, int flags);
669
670int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
671int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
672int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
673 u64 flags);
674int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
675 u64 flags);
676
677int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
678
679int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
680 void *key, void *value, u64 map_flags);
681int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
682int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
683 void *key, void *value, u64 map_flags);
684int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
685
686int bpf_get_file_flag(int flags);
687int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size,
688 size_t actual_size);
689
690
691
692
693
694
695
696static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
697{
698 const long *lsrc = src;
699 long *ldst = dst;
700
701 size /= sizeof(long);
702 while (size--)
703 *ldst++ = *lsrc++;
704}
705
706
707int bpf_check(struct bpf_prog **fp, union bpf_attr *attr,
708 union bpf_attr __user *uattr);
709void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
710
711
712struct xdp_buff;
713struct sk_buff;
714
715struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key);
716void __dev_map_insert_ctx(struct bpf_map *map, u32 index);
717void __dev_map_flush(struct bpf_map *map);
718int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,
719 struct net_device *dev_rx);
720int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
721 struct bpf_prog *xdp_prog);
722
723struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key);
724void __cpu_map_insert_ctx(struct bpf_map *map, u32 index);
725void __cpu_map_flush(struct bpf_map *map);
726int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
727 struct net_device *dev_rx);
728
729
730static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
731{
732 return (attr->map_flags & BPF_F_NUMA_NODE) ?
733 attr->numa_node : NUMA_NO_NODE;
734}
735
736struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type);
737int array_map_alloc_check(union bpf_attr *attr);
738
739int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
740 union bpf_attr __user *uattr);
741int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
742 union bpf_attr __user *uattr);
743int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
744 const union bpf_attr *kattr,
745 union bpf_attr __user *uattr);
746#else
747static inline struct bpf_prog *bpf_prog_get(u32 ufd)
748{
749 return ERR_PTR(-EOPNOTSUPP);
750}
751
752static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
753 enum bpf_prog_type type,
754 bool attach_drv)
755{
756 return ERR_PTR(-EOPNOTSUPP);
757}
758
759static inline struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog,
760 int i)
761{
762 return ERR_PTR(-EOPNOTSUPP);
763}
764
765static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
766{
767}
768
769static inline void bpf_prog_put(struct bpf_prog *prog)
770{
771}
772
773static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog)
774{
775 return ERR_PTR(-EOPNOTSUPP);
776}
777
778static inline struct bpf_prog *__must_check
779bpf_prog_inc_not_zero(struct bpf_prog *prog)
780{
781 return ERR_PTR(-EOPNOTSUPP);
782}
783
784static inline int __bpf_prog_charge(struct user_struct *user, u32 pages)
785{
786 return 0;
787}
788
789static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
790{
791}
792
793static inline int bpf_obj_get_user(const char __user *pathname, int flags)
794{
795 return -EOPNOTSUPP;
796}
797
798static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map,
799 u32 key)
800{
801 return NULL;
802}
803
804static inline void __dev_map_insert_ctx(struct bpf_map *map, u32 index)
805{
806}
807
808static inline void __dev_map_flush(struct bpf_map *map)
809{
810}
811
812struct xdp_buff;
813struct bpf_dtab_netdev;
814
815static inline
816int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,
817 struct net_device *dev_rx)
818{
819 return 0;
820}
821
822struct sk_buff;
823
824static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
825 struct sk_buff *skb,
826 struct bpf_prog *xdp_prog)
827{
828 return 0;
829}
830
831static inline
832struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
833{
834 return NULL;
835}
836
837static inline void __cpu_map_insert_ctx(struct bpf_map *map, u32 index)
838{
839}
840
841static inline void __cpu_map_flush(struct bpf_map *map)
842{
843}
844
845static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
846 struct xdp_buff *xdp,
847 struct net_device *dev_rx)
848{
849 return 0;
850}
851
852static inline struct bpf_prog *bpf_prog_get_type_path(const char *name,
853 enum bpf_prog_type type)
854{
855 return ERR_PTR(-EOPNOTSUPP);
856}
857
858static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,
859 const union bpf_attr *kattr,
860 union bpf_attr __user *uattr)
861{
862 return -ENOTSUPP;
863}
864
865static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,
866 const union bpf_attr *kattr,
867 union bpf_attr __user *uattr)
868{
869 return -ENOTSUPP;
870}
871
872static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
873 const union bpf_attr *kattr,
874 union bpf_attr __user *uattr)
875{
876 return -ENOTSUPP;
877}
878#endif
879
880static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
881 enum bpf_prog_type type)
882{
883 return bpf_prog_get_type_dev(ufd, type, false);
884}
885
886bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);
887
888int bpf_prog_offload_compile(struct bpf_prog *prog);
889void bpf_prog_offload_destroy(struct bpf_prog *prog);
890int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
891 struct bpf_prog *prog);
892
893int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map);
894
895int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value);
896int bpf_map_offload_update_elem(struct bpf_map *map,
897 void *key, void *value, u64 flags);
898int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
899int bpf_map_offload_get_next_key(struct bpf_map *map,
900 void *key, void *next_key);
901
902bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);
903
904struct bpf_offload_dev *
905bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv);
906void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
907void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev);
908int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
909 struct net_device *netdev);
910void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
911 struct net_device *netdev);
912bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
913
914#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
915int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr);
916
917static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
918{
919 return aux->offload_requested;
920}
921
922static inline bool bpf_map_is_dev_bound(struct bpf_map *map)
923{
924 return unlikely(map->ops == &bpf_map_offload_ops);
925}
926
927struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);
928void bpf_map_offload_map_free(struct bpf_map *map);
929#else
930static inline int bpf_prog_offload_init(struct bpf_prog *prog,
931 union bpf_attr *attr)
932{
933 return -EOPNOTSUPP;
934}
935
936static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux)
937{
938 return false;
939}
940
941static inline bool bpf_map_is_dev_bound(struct bpf_map *map)
942{
943 return false;
944}
945
946static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
947{
948 return ERR_PTR(-EOPNOTSUPP);
949}
950
951static inline void bpf_map_offload_map_free(struct bpf_map *map)
952{
953}
954#endif
955
956#if defined(CONFIG_BPF_STREAM_PARSER)
957int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, u32 which);
958int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog);
959#else
960static inline int sock_map_prog_update(struct bpf_map *map,
961 struct bpf_prog *prog, u32 which)
962{
963 return -EOPNOTSUPP;
964}
965
966static inline int sock_map_get_from_fd(const union bpf_attr *attr,
967 struct bpf_prog *prog)
968{
969 return -EINVAL;
970}
971#endif
972
973#if defined(CONFIG_XDP_SOCKETS)
974struct xdp_sock;
975struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key);
976int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp,
977 struct xdp_sock *xs);
978void __xsk_map_flush(struct bpf_map *map);
979#else
980struct xdp_sock;
981static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map,
982 u32 key)
983{
984 return NULL;
985}
986
987static inline int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp,
988 struct xdp_sock *xs)
989{
990 return -EOPNOTSUPP;
991}
992
993static inline void __xsk_map_flush(struct bpf_map *map)
994{
995}
996#endif
997
998#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
999void bpf_sk_reuseport_detach(struct sock *sk);
1000int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
1001 void *value);
1002int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
1003 void *value, u64 map_flags);
1004#else
1005static inline void bpf_sk_reuseport_detach(struct sock *sk)
1006{
1007}
1008
1009#ifdef CONFIG_BPF_SYSCALL
1010static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,
1011 void *key, void *value)
1012{
1013 return -EOPNOTSUPP;
1014}
1015
1016static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
1017 void *key, void *value,
1018 u64 map_flags)
1019{
1020 return -EOPNOTSUPP;
1021}
1022#endif
1023#endif
1024
1025
1026extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
1027extern const struct bpf_func_proto bpf_map_update_elem_proto;
1028extern const struct bpf_func_proto bpf_map_delete_elem_proto;
1029extern const struct bpf_func_proto bpf_map_push_elem_proto;
1030extern const struct bpf_func_proto bpf_map_pop_elem_proto;
1031extern const struct bpf_func_proto bpf_map_peek_elem_proto;
1032
1033extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
1034extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
1035extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
1036extern const struct bpf_func_proto bpf_tail_call_proto;
1037extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
1038extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
1039extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
1040extern const struct bpf_func_proto bpf_get_current_comm_proto;
1041extern const struct bpf_func_proto bpf_get_stackid_proto;
1042extern const struct bpf_func_proto bpf_get_stack_proto;
1043extern const struct bpf_func_proto bpf_sock_map_update_proto;
1044extern const struct bpf_func_proto bpf_sock_hash_update_proto;
1045extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
1046extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
1047extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
1048extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
1049extern const struct bpf_func_proto bpf_sk_redirect_map_proto;
1050extern const struct bpf_func_proto bpf_spin_lock_proto;
1051extern const struct bpf_func_proto bpf_spin_unlock_proto;
1052extern const struct bpf_func_proto bpf_get_local_storage_proto;
1053extern const struct bpf_func_proto bpf_strtol_proto;
1054extern const struct bpf_func_proto bpf_strtoul_proto;
1055extern const struct bpf_func_proto bpf_tcp_sock_proto;
1056
1057
1058void bpf_user_rnd_init_once(void);
1059u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
1060
1061#if defined(CONFIG_NET)
1062bool bpf_sock_common_is_valid_access(int off, int size,
1063 enum bpf_access_type type,
1064 struct bpf_insn_access_aux *info);
1065bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
1066 struct bpf_insn_access_aux *info);
1067u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
1068 const struct bpf_insn *si,
1069 struct bpf_insn *insn_buf,
1070 struct bpf_prog *prog,
1071 u32 *target_size);
1072#else
1073static inline bool bpf_sock_common_is_valid_access(int off, int size,
1074 enum bpf_access_type type,
1075 struct bpf_insn_access_aux *info)
1076{
1077 return false;
1078}
1079static inline bool bpf_sock_is_valid_access(int off, int size,
1080 enum bpf_access_type type,
1081 struct bpf_insn_access_aux *info)
1082{
1083 return false;
1084}
1085static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
1086 const struct bpf_insn *si,
1087 struct bpf_insn *insn_buf,
1088 struct bpf_prog *prog,
1089 u32 *target_size)
1090{
1091 return 0;
1092}
1093#endif
1094
1095#ifdef CONFIG_INET
1096bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
1097 struct bpf_insn_access_aux *info);
1098
1099u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
1100 const struct bpf_insn *si,
1101 struct bpf_insn *insn_buf,
1102 struct bpf_prog *prog,
1103 u32 *target_size);
1104
1105bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
1106 struct bpf_insn_access_aux *info);
1107
1108u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
1109 const struct bpf_insn *si,
1110 struct bpf_insn *insn_buf,
1111 struct bpf_prog *prog,
1112 u32 *target_size);
1113#else
1114static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
1115 enum bpf_access_type type,
1116 struct bpf_insn_access_aux *info)
1117{
1118 return false;
1119}
1120
1121static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
1122 const struct bpf_insn *si,
1123 struct bpf_insn *insn_buf,
1124 struct bpf_prog *prog,
1125 u32 *target_size)
1126{
1127 return 0;
1128}
1129static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
1130 enum bpf_access_type type,
1131 struct bpf_insn_access_aux *info)
1132{
1133 return false;
1134}
1135
1136static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
1137 const struct bpf_insn *si,
1138 struct bpf_insn *insn_buf,
1139 struct bpf_prog *prog,
1140 u32 *target_size)
1141{
1142 return 0;
1143}
1144#endif
1145
1146#endif
1147