1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/atomic.h>
21#include <linux/module.h>
22#include <linux/types.h>
23#include <linux/mm.h>
24#include <linux/fcntl.h>
25#include <linux/socket.h>
26#include <linux/sock_diag.h>
27#include <linux/in.h>
28#include <linux/inet.h>
29#include <linux/netdevice.h>
30#include <linux/if_packet.h>
31#include <linux/if_arp.h>
32#include <linux/gfp.h>
33#include <net/inet_common.h>
34#include <net/ip.h>
35#include <net/protocol.h>
36#include <net/netlink.h>
37#include <linux/skbuff.h>
38#include <linux/skmsg.h>
39#include <net/sock.h>
40#include <net/flow_dissector.h>
41#include <linux/errno.h>
42#include <linux/timer.h>
43#include <linux/uaccess.h>
44#include <asm/unaligned.h>
45#include <linux/filter.h>
46#include <linux/ratelimit.h>
47#include <linux/seccomp.h>
48#include <linux/if_vlan.h>
49#include <linux/bpf.h>
50#include <linux/btf.h>
51#include <net/sch_generic.h>
52#include <net/cls_cgroup.h>
53#include <net/dst_metadata.h>
54#include <net/dst.h>
55#include <net/sock_reuseport.h>
56#include <net/busy_poll.h>
57#include <net/tcp.h>
58#include <net/xfrm.h>
59#include <net/udp.h>
60#include <linux/bpf_trace.h>
61#include <net/xdp_sock.h>
62#include <linux/inetdevice.h>
63#include <net/inet_hashtables.h>
64#include <net/inet6_hashtables.h>
65#include <net/ip_fib.h>
66#include <net/nexthop.h>
67#include <net/flow.h>
68#include <net/arp.h>
69#include <net/ipv6.h>
70#include <net/net_namespace.h>
71#include <linux/seg6_local.h>
72#include <net/seg6.h>
73#include <net/seg6_local.h>
74#include <net/lwtunnel.h>
75#include <net/ipv6_stubs.h>
76#include <net/bpf_sk_storage.h>
77#include <net/transp_v6.h>
78#include <linux/btf_ids.h>
79#include <net/tls.h>
80
81static const struct bpf_func_proto *
82bpf_sk_base_func_proto(enum bpf_func_id func_id);
83
84int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len)
85{
86 if (in_compat_syscall()) {
87 struct compat_sock_fprog f32;
88
89 if (len != sizeof(f32))
90 return -EINVAL;
91 if (copy_from_sockptr(&f32, src, sizeof(f32)))
92 return -EFAULT;
93 memset(dst, 0, sizeof(*dst));
94 dst->len = f32.len;
95 dst->filter = compat_ptr(f32.filter);
96 } else {
97 if (len != sizeof(*dst))
98 return -EINVAL;
99 if (copy_from_sockptr(dst, src, sizeof(*dst)))
100 return -EFAULT;
101 }
102
103 return 0;
104}
105EXPORT_SYMBOL_GPL(copy_bpf_fprog_from_user);
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap)
121{
122 int err;
123 struct sk_filter *filter;
124
125
126
127
128
129
130 if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC)) {
131 NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP);
132 return -ENOMEM;
133 }
134 err = BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb);
135 if (err)
136 return err;
137
138 err = security_sock_rcv_skb(sk, skb);
139 if (err)
140 return err;
141
142 rcu_read_lock();
143 filter = rcu_dereference(sk->sk_filter);
144 if (filter) {
145 struct sock *save_sk = skb->sk;
146 unsigned int pkt_len;
147
148 skb->sk = sk;
149 pkt_len = bpf_prog_run_save_cb(filter->prog, skb);
150 skb->sk = save_sk;
151 err = pkt_len ? pskb_trim(skb, max(cap, pkt_len)) : -EPERM;
152 }
153 rcu_read_unlock();
154
155 return err;
156}
157EXPORT_SYMBOL(sk_filter_trim_cap);
158
159BPF_CALL_1(bpf_skb_get_pay_offset, struct sk_buff *, skb)
160{
161 return skb_get_poff(skb);
162}
163
164BPF_CALL_3(bpf_skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x)
165{
166 struct nlattr *nla;
167
168 if (skb_is_nonlinear(skb))
169 return 0;
170
171 if (skb->len < sizeof(struct nlattr))
172 return 0;
173
174 if (a > skb->len - sizeof(struct nlattr))
175 return 0;
176
177 nla = nla_find((struct nlattr *) &skb->data[a], skb->len - a, x);
178 if (nla)
179 return (void *) nla - (void *) skb->data;
180
181 return 0;
182}
183
184BPF_CALL_3(bpf_skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x)
185{
186 struct nlattr *nla;
187
188 if (skb_is_nonlinear(skb))
189 return 0;
190
191 if (skb->len < sizeof(struct nlattr))
192 return 0;
193
194 if (a > skb->len - sizeof(struct nlattr))
195 return 0;
196
197 nla = (struct nlattr *) &skb->data[a];
198 if (nla->nla_len > skb->len - a)
199 return 0;
200
201 nla = nla_find_nested(nla, x);
202 if (nla)
203 return (void *) nla - (void *) skb->data;
204
205 return 0;
206}
207
208BPF_CALL_4(bpf_skb_load_helper_8, const struct sk_buff *, skb, const void *,
209 data, int, headlen, int, offset)
210{
211 u8 tmp, *ptr;
212 const int len = sizeof(tmp);
213
214 if (offset >= 0) {
215 if (headlen - offset >= len)
216 return *(u8 *)(data + offset);
217 if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
218 return tmp;
219 } else {
220 ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len);
221 if (likely(ptr))
222 return *(u8 *)ptr;
223 }
224
225 return -EFAULT;
226}
227
228BPF_CALL_2(bpf_skb_load_helper_8_no_cache, const struct sk_buff *, skb,
229 int, offset)
230{
231 return ____bpf_skb_load_helper_8(skb, skb->data, skb->len - skb->data_len,
232 offset);
233}
234
235BPF_CALL_4(bpf_skb_load_helper_16, const struct sk_buff *, skb, const void *,
236 data, int, headlen, int, offset)
237{
238 u16 tmp, *ptr;
239 const int len = sizeof(tmp);
240
241 if (offset >= 0) {
242 if (headlen - offset >= len)
243 return get_unaligned_be16(data + offset);
244 if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
245 return be16_to_cpu(tmp);
246 } else {
247 ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len);
248 if (likely(ptr))
249 return get_unaligned_be16(ptr);
250 }
251
252 return -EFAULT;
253}
254
255BPF_CALL_2(bpf_skb_load_helper_16_no_cache, const struct sk_buff *, skb,
256 int, offset)
257{
258 return ____bpf_skb_load_helper_16(skb, skb->data, skb->len - skb->data_len,
259 offset);
260}
261
262BPF_CALL_4(bpf_skb_load_helper_32, const struct sk_buff *, skb, const void *,
263 data, int, headlen, int, offset)
264{
265 u32 tmp, *ptr;
266 const int len = sizeof(tmp);
267
268 if (likely(offset >= 0)) {
269 if (headlen - offset >= len)
270 return get_unaligned_be32(data + offset);
271 if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
272 return be32_to_cpu(tmp);
273 } else {
274 ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len);
275 if (likely(ptr))
276 return get_unaligned_be32(ptr);
277 }
278
279 return -EFAULT;
280}
281
282BPF_CALL_2(bpf_skb_load_helper_32_no_cache, const struct sk_buff *, skb,
283 int, offset)
284{
285 return ____bpf_skb_load_helper_32(skb, skb->data, skb->len - skb->data_len,
286 offset);
287}
288
289static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg,
290 struct bpf_insn *insn_buf)
291{
292 struct bpf_insn *insn = insn_buf;
293
294 switch (skb_field) {
295 case SKF_AD_MARK:
296 BUILD_BUG_ON(sizeof_field(struct sk_buff, mark) != 4);
297
298 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
299 offsetof(struct sk_buff, mark));
300 break;
301
302 case SKF_AD_PKTTYPE:
303 *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET());
304 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX);
305#ifdef __BIG_ENDIAN_BITFIELD
306 *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5);
307#endif
308 break;
309
310 case SKF_AD_QUEUE:
311 BUILD_BUG_ON(sizeof_field(struct sk_buff, queue_mapping) != 2);
312
313 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
314 offsetof(struct sk_buff, queue_mapping));
315 break;
316
317 case SKF_AD_VLAN_TAG:
318 BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_tci) != 2);
319
320
321 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
322 offsetof(struct sk_buff, vlan_tci));
323 break;
324 case SKF_AD_VLAN_TAG_PRESENT:
325 *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_VLAN_PRESENT_OFFSET());
326 if (PKT_VLAN_PRESENT_BIT)
327 *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, PKT_VLAN_PRESENT_BIT);
328 if (PKT_VLAN_PRESENT_BIT < 7)
329 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, 1);
330 break;
331 }
332
333 return insn - insn_buf;
334}
335
336static bool convert_bpf_extensions(struct sock_filter *fp,
337 struct bpf_insn **insnp)
338{
339 struct bpf_insn *insn = *insnp;
340 u32 cnt;
341
342 switch (fp->k) {
343 case SKF_AD_OFF + SKF_AD_PROTOCOL:
344 BUILD_BUG_ON(sizeof_field(struct sk_buff, protocol) != 2);
345
346
347 *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
348 offsetof(struct sk_buff, protocol));
349
350 *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16);
351 break;
352
353 case SKF_AD_OFF + SKF_AD_PKTTYPE:
354 cnt = convert_skb_access(SKF_AD_PKTTYPE, BPF_REG_A, BPF_REG_CTX, insn);
355 insn += cnt - 1;
356 break;
357
358 case SKF_AD_OFF + SKF_AD_IFINDEX:
359 case SKF_AD_OFF + SKF_AD_HATYPE:
360 BUILD_BUG_ON(sizeof_field(struct net_device, ifindex) != 4);
361 BUILD_BUG_ON(sizeof_field(struct net_device, type) != 2);
362
363 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
364 BPF_REG_TMP, BPF_REG_CTX,
365 offsetof(struct sk_buff, dev));
366
367 *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_TMP, 0, 1);
368 *insn++ = BPF_EXIT_INSN();
369 if (fp->k == SKF_AD_OFF + SKF_AD_IFINDEX)
370 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_TMP,
371 offsetof(struct net_device, ifindex));
372 else
373 *insn = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_TMP,
374 offsetof(struct net_device, type));
375 break;
376
377 case SKF_AD_OFF + SKF_AD_MARK:
378 cnt = convert_skb_access(SKF_AD_MARK, BPF_REG_A, BPF_REG_CTX, insn);
379 insn += cnt - 1;
380 break;
381
382 case SKF_AD_OFF + SKF_AD_RXHASH:
383 BUILD_BUG_ON(sizeof_field(struct sk_buff, hash) != 4);
384
385 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX,
386 offsetof(struct sk_buff, hash));
387 break;
388
389 case SKF_AD_OFF + SKF_AD_QUEUE:
390 cnt = convert_skb_access(SKF_AD_QUEUE, BPF_REG_A, BPF_REG_CTX, insn);
391 insn += cnt - 1;
392 break;
393
394 case SKF_AD_OFF + SKF_AD_VLAN_TAG:
395 cnt = convert_skb_access(SKF_AD_VLAN_TAG,
396 BPF_REG_A, BPF_REG_CTX, insn);
397 insn += cnt - 1;
398 break;
399
400 case SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT:
401 cnt = convert_skb_access(SKF_AD_VLAN_TAG_PRESENT,
402 BPF_REG_A, BPF_REG_CTX, insn);
403 insn += cnt - 1;
404 break;
405
406 case SKF_AD_OFF + SKF_AD_VLAN_TPID:
407 BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_proto) != 2);
408
409
410 *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
411 offsetof(struct sk_buff, vlan_proto));
412
413 *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16);
414 break;
415
416 case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
417 case SKF_AD_OFF + SKF_AD_NLATTR:
418 case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
419 case SKF_AD_OFF + SKF_AD_CPU:
420 case SKF_AD_OFF + SKF_AD_RANDOM:
421
422 *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX);
423
424 *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_A);
425
426 *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_X);
427
428 switch (fp->k) {
429 case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
430 *insn = BPF_EMIT_CALL(bpf_skb_get_pay_offset);
431 break;
432 case SKF_AD_OFF + SKF_AD_NLATTR:
433 *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr);
434 break;
435 case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
436 *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr_nest);
437 break;
438 case SKF_AD_OFF + SKF_AD_CPU:
439 *insn = BPF_EMIT_CALL(bpf_get_raw_cpu_id);
440 break;
441 case SKF_AD_OFF + SKF_AD_RANDOM:
442 *insn = BPF_EMIT_CALL(bpf_user_rnd_u32);
443 bpf_user_rnd_init_once();
444 break;
445 }
446 break;
447
448 case SKF_AD_OFF + SKF_AD_ALU_XOR_X:
449
450 *insn = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_X);
451 break;
452
453 default:
454
455
456
457
458 BUG_ON(__bpf_call_base(0, 0, 0, 0, 0) != 0);
459 return false;
460 }
461
462 *insnp = insn;
463 return true;
464}
465
466static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp)
467{
468 const bool unaligned_ok = IS_BUILTIN(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS);
469 int size = bpf_size_to_bytes(BPF_SIZE(fp->code));
470 bool endian = BPF_SIZE(fp->code) == BPF_H ||
471 BPF_SIZE(fp->code) == BPF_W;
472 bool indirect = BPF_MODE(fp->code) == BPF_IND;
473 const int ip_align = NET_IP_ALIGN;
474 struct bpf_insn *insn = *insnp;
475 int offset = fp->k;
476
477 if (!indirect &&
478 ((unaligned_ok && offset >= 0) ||
479 (!unaligned_ok && offset >= 0 &&
480 offset + ip_align >= 0 &&
481 offset + ip_align % size == 0))) {
482 bool ldx_off_ok = offset <= S16_MAX;
483
484 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);
485 if (offset)
486 *insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);
487 *insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP,
488 size, 2 + endian + (!ldx_off_ok * 2));
489 if (ldx_off_ok) {
490 *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A,
491 BPF_REG_D, offset);
492 } else {
493 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_D);
494 *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_TMP, offset);
495 *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A,
496 BPF_REG_TMP, 0);
497 }
498 if (endian)
499 *insn++ = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, size * 8);
500 *insn++ = BPF_JMP_A(8);
501 }
502
503 *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX);
504 *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_D);
505 *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_H);
506 if (!indirect) {
507 *insn++ = BPF_MOV64_IMM(BPF_REG_ARG4, offset);
508 } else {
509 *insn++ = BPF_MOV64_REG(BPF_REG_ARG4, BPF_REG_X);
510 if (fp->k)
511 *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG4, offset);
512 }
513
514 switch (BPF_SIZE(fp->code)) {
515 case BPF_B:
516 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8);
517 break;
518 case BPF_H:
519 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16);
520 break;
521 case BPF_W:
522 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32);
523 break;
524 default:
525 return false;
526 }
527
528 *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_A, 0, 2);
529 *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
530 *insn = BPF_EXIT_INSN();
531
532 *insnp = insn;
533 return true;
534}
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555static int bpf_convert_filter(struct sock_filter *prog, int len,
556 struct bpf_prog *new_prog, int *new_len,
557 bool *seen_ld_abs)
558{
559 int new_flen = 0, pass = 0, target, i, stack_off;
560 struct bpf_insn *new_insn, *first_insn = NULL;
561 struct sock_filter *fp;
562 int *addrs = NULL;
563 u8 bpf_src;
564
565 BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK);
566 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
567
568 if (len <= 0 || len > BPF_MAXINSNS)
569 return -EINVAL;
570
571 if (new_prog) {
572 first_insn = new_prog->insnsi;
573 addrs = kcalloc(len, sizeof(*addrs),
574 GFP_KERNEL | __GFP_NOWARN);
575 if (!addrs)
576 return -ENOMEM;
577 }
578
579do_pass:
580 new_insn = first_insn;
581 fp = prog;
582
583
584 if (new_prog) {
585
586
587
588 *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
589 *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_X, BPF_REG_X);
590
591
592
593
594
595 *new_insn++ = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1);
596 if (*seen_ld_abs) {
597
598
599
600
601
602 *new_insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
603 BPF_REG_D, BPF_REG_CTX,
604 offsetof(struct sk_buff, data));
605 *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_H, BPF_REG_CTX,
606 offsetof(struct sk_buff, len));
607 *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_TMP, BPF_REG_CTX,
608 offsetof(struct sk_buff, data_len));
609 *new_insn++ = BPF_ALU32_REG(BPF_SUB, BPF_REG_H, BPF_REG_TMP);
610 }
611 } else {
612 new_insn += 3;
613 }
614
615 for (i = 0; i < len; fp++, i++) {
616 struct bpf_insn tmp_insns[32] = { };
617 struct bpf_insn *insn = tmp_insns;
618
619 if (addrs)
620 addrs[i] = new_insn - first_insn;
621
622 switch (fp->code) {
623
624 case BPF_ALU | BPF_ADD | BPF_X:
625 case BPF_ALU | BPF_ADD | BPF_K:
626 case BPF_ALU | BPF_SUB | BPF_X:
627 case BPF_ALU | BPF_SUB | BPF_K:
628 case BPF_ALU | BPF_AND | BPF_X:
629 case BPF_ALU | BPF_AND | BPF_K:
630 case BPF_ALU | BPF_OR | BPF_X:
631 case BPF_ALU | BPF_OR | BPF_K:
632 case BPF_ALU | BPF_LSH | BPF_X:
633 case BPF_ALU | BPF_LSH | BPF_K:
634 case BPF_ALU | BPF_RSH | BPF_X:
635 case BPF_ALU | BPF_RSH | BPF_K:
636 case BPF_ALU | BPF_XOR | BPF_X:
637 case BPF_ALU | BPF_XOR | BPF_K:
638 case BPF_ALU | BPF_MUL | BPF_X:
639 case BPF_ALU | BPF_MUL | BPF_K:
640 case BPF_ALU | BPF_DIV | BPF_X:
641 case BPF_ALU | BPF_DIV | BPF_K:
642 case BPF_ALU | BPF_MOD | BPF_X:
643 case BPF_ALU | BPF_MOD | BPF_K:
644 case BPF_ALU | BPF_NEG:
645 case BPF_LD | BPF_ABS | BPF_W:
646 case BPF_LD | BPF_ABS | BPF_H:
647 case BPF_LD | BPF_ABS | BPF_B:
648 case BPF_LD | BPF_IND | BPF_W:
649 case BPF_LD | BPF_IND | BPF_H:
650 case BPF_LD | BPF_IND | BPF_B:
651
652
653
654
655 if (BPF_CLASS(fp->code) == BPF_LD &&
656 BPF_MODE(fp->code) == BPF_ABS &&
657 convert_bpf_extensions(fp, &insn))
658 break;
659 if (BPF_CLASS(fp->code) == BPF_LD &&
660 convert_bpf_ld_abs(fp, &insn)) {
661 *seen_ld_abs = true;
662 break;
663 }
664
665 if (fp->code == (BPF_ALU | BPF_DIV | BPF_X) ||
666 fp->code == (BPF_ALU | BPF_MOD | BPF_X)) {
667 *insn++ = BPF_MOV32_REG(BPF_REG_X, BPF_REG_X);
668
669
670
671 *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_X, 0, 2);
672 *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
673 *insn++ = BPF_EXIT_INSN();
674 }
675
676 *insn = BPF_RAW_INSN(fp->code, BPF_REG_A, BPF_REG_X, 0, fp->k);
677 break;
678
679
680
681
682
683
684
685#define BPF_EMIT_JMP \
686 do { \
687 const s32 off_min = S16_MIN, off_max = S16_MAX; \
688 s32 off; \
689 \
690 if (target >= len || target < 0) \
691 goto err; \
692 off = addrs ? addrs[target] - addrs[i] - 1 : 0; \
693 \
694 off -= insn - tmp_insns; \
695 \
696 if (off < off_min || off > off_max) \
697 goto err; \
698 insn->off = off; \
699 } while (0)
700
701 case BPF_JMP | BPF_JA:
702 target = i + fp->k + 1;
703 insn->code = fp->code;
704 BPF_EMIT_JMP;
705 break;
706
707 case BPF_JMP | BPF_JEQ | BPF_K:
708 case BPF_JMP | BPF_JEQ | BPF_X:
709 case BPF_JMP | BPF_JSET | BPF_K:
710 case BPF_JMP | BPF_JSET | BPF_X:
711 case BPF_JMP | BPF_JGT | BPF_K:
712 case BPF_JMP | BPF_JGT | BPF_X:
713 case BPF_JMP | BPF_JGE | BPF_K:
714 case BPF_JMP | BPF_JGE | BPF_X:
715 if (BPF_SRC(fp->code) == BPF_K && (int) fp->k < 0) {
716
717
718
719
720 *insn++ = BPF_MOV32_IMM(BPF_REG_TMP, fp->k);
721
722 insn->dst_reg = BPF_REG_A;
723 insn->src_reg = BPF_REG_TMP;
724 bpf_src = BPF_X;
725 } else {
726 insn->dst_reg = BPF_REG_A;
727 insn->imm = fp->k;
728 bpf_src = BPF_SRC(fp->code);
729 insn->src_reg = bpf_src == BPF_X ? BPF_REG_X : 0;
730 }
731
732
733 if (fp->jf == 0) {
734 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
735 target = i + fp->jt + 1;
736 BPF_EMIT_JMP;
737 break;
738 }
739
740
741 if (fp->jt == 0) {
742 switch (BPF_OP(fp->code)) {
743 case BPF_JEQ:
744 insn->code = BPF_JMP | BPF_JNE | bpf_src;
745 break;
746 case BPF_JGT:
747 insn->code = BPF_JMP | BPF_JLE | bpf_src;
748 break;
749 case BPF_JGE:
750 insn->code = BPF_JMP | BPF_JLT | bpf_src;
751 break;
752 default:
753 goto jmp_rest;
754 }
755
756 target = i + fp->jf + 1;
757 BPF_EMIT_JMP;
758 break;
759 }
760jmp_rest:
761
762 target = i + fp->jt + 1;
763 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
764 BPF_EMIT_JMP;
765 insn++;
766
767 insn->code = BPF_JMP | BPF_JA;
768 target = i + fp->jf + 1;
769 BPF_EMIT_JMP;
770 break;
771
772
773 case BPF_LDX | BPF_MSH | BPF_B: {
774 struct sock_filter tmp = {
775 .code = BPF_LD | BPF_ABS | BPF_B,
776 .k = fp->k,
777 };
778
779 *seen_ld_abs = true;
780
781
782 *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
783
784 convert_bpf_ld_abs(&tmp, &insn);
785 insn++;
786
787 *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, 0xf);
788
789 *insn++ = BPF_ALU32_IMM(BPF_LSH, BPF_REG_A, 2);
790
791 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_X);
792
793 *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
794
795 *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_TMP);
796 break;
797 }
798
799
800
801 case BPF_RET | BPF_A:
802 case BPF_RET | BPF_K:
803 if (BPF_RVAL(fp->code) == BPF_K)
804 *insn++ = BPF_MOV32_RAW(BPF_K, BPF_REG_0,
805 0, fp->k);
806 *insn = BPF_EXIT_INSN();
807 break;
808
809
810 case BPF_ST:
811 case BPF_STX:
812 stack_off = fp->k * 4 + 4;
813 *insn = BPF_STX_MEM(BPF_W, BPF_REG_FP, BPF_CLASS(fp->code) ==
814 BPF_ST ? BPF_REG_A : BPF_REG_X,
815 -stack_off);
816
817
818
819
820 if (new_prog && new_prog->aux->stack_depth < stack_off)
821 new_prog->aux->stack_depth = stack_off;
822 break;
823
824
825 case BPF_LD | BPF_MEM:
826 case BPF_LDX | BPF_MEM:
827 stack_off = fp->k * 4 + 4;
828 *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ?
829 BPF_REG_A : BPF_REG_X, BPF_REG_FP,
830 -stack_off);
831 break;
832
833
834 case BPF_LD | BPF_IMM:
835 case BPF_LDX | BPF_IMM:
836 *insn = BPF_MOV32_IMM(BPF_CLASS(fp->code) == BPF_LD ?
837 BPF_REG_A : BPF_REG_X, fp->k);
838 break;
839
840
841 case BPF_MISC | BPF_TAX:
842 *insn = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
843 break;
844
845
846 case BPF_MISC | BPF_TXA:
847 *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_X);
848 break;
849
850
851 case BPF_LD | BPF_W | BPF_LEN:
852 case BPF_LDX | BPF_W | BPF_LEN:
853 *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ?
854 BPF_REG_A : BPF_REG_X, BPF_REG_CTX,
855 offsetof(struct sk_buff, len));
856 break;
857
858
859 case BPF_LDX | BPF_ABS | BPF_W:
860
861 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX, fp->k);
862 break;
863
864
865 default:
866 goto err;
867 }
868
869 insn++;
870 if (new_prog)
871 memcpy(new_insn, tmp_insns,
872 sizeof(*insn) * (insn - tmp_insns));
873 new_insn += insn - tmp_insns;
874 }
875
876 if (!new_prog) {
877
878 *new_len = new_insn - first_insn;
879 if (*seen_ld_abs)
880 *new_len += 4;
881 return 0;
882 }
883
884 pass++;
885 if (new_flen != new_insn - first_insn) {
886 new_flen = new_insn - first_insn;
887 if (pass > 2)
888 goto err;
889 goto do_pass;
890 }
891
892 kfree(addrs);
893 BUG_ON(*new_len != new_flen);
894 return 0;
895err:
896 kfree(addrs);
897 return -EINVAL;
898}
899
900
901
902
903
904
905
906
907static int check_load_and_stores(const struct sock_filter *filter, int flen)
908{
909 u16 *masks, memvalid = 0;
910 int pc, ret = 0;
911
912 BUILD_BUG_ON(BPF_MEMWORDS > 16);
913
914 masks = kmalloc_array(flen, sizeof(*masks), GFP_KERNEL);
915 if (!masks)
916 return -ENOMEM;
917
918 memset(masks, 0xff, flen * sizeof(*masks));
919
920 for (pc = 0; pc < flen; pc++) {
921 memvalid &= masks[pc];
922
923 switch (filter[pc].code) {
924 case BPF_ST:
925 case BPF_STX:
926 memvalid |= (1 << filter[pc].k);
927 break;
928 case BPF_LD | BPF_MEM:
929 case BPF_LDX | BPF_MEM:
930 if (!(memvalid & (1 << filter[pc].k))) {
931 ret = -EINVAL;
932 goto error;
933 }
934 break;
935 case BPF_JMP | BPF_JA:
936
937 masks[pc + 1 + filter[pc].k] &= memvalid;
938 memvalid = ~0;
939 break;
940 case BPF_JMP | BPF_JEQ | BPF_K:
941 case BPF_JMP | BPF_JEQ | BPF_X:
942 case BPF_JMP | BPF_JGE | BPF_K:
943 case BPF_JMP | BPF_JGE | BPF_X:
944 case BPF_JMP | BPF_JGT | BPF_K:
945 case BPF_JMP | BPF_JGT | BPF_X:
946 case BPF_JMP | BPF_JSET | BPF_K:
947 case BPF_JMP | BPF_JSET | BPF_X:
948
949 masks[pc + 1 + filter[pc].jt] &= memvalid;
950 masks[pc + 1 + filter[pc].jf] &= memvalid;
951 memvalid = ~0;
952 break;
953 }
954 }
955error:
956 kfree(masks);
957 return ret;
958}
959
960static bool chk_code_allowed(u16 code_to_probe)
961{
962 static const bool codes[] = {
963
964 [BPF_ALU | BPF_ADD | BPF_K] = true,
965 [BPF_ALU | BPF_ADD | BPF_X] = true,
966 [BPF_ALU | BPF_SUB | BPF_K] = true,
967 [BPF_ALU | BPF_SUB | BPF_X] = true,
968 [BPF_ALU | BPF_MUL | BPF_K] = true,
969 [BPF_ALU | BPF_MUL | BPF_X] = true,
970 [BPF_ALU | BPF_DIV | BPF_K] = true,
971 [BPF_ALU | BPF_DIV | BPF_X] = true,
972 [BPF_ALU | BPF_MOD | BPF_K] = true,
973 [BPF_ALU | BPF_MOD | BPF_X] = true,
974 [BPF_ALU | BPF_AND | BPF_K] = true,
975 [BPF_ALU | BPF_AND | BPF_X] = true,
976 [BPF_ALU | BPF_OR | BPF_K] = true,
977 [BPF_ALU | BPF_OR | BPF_X] = true,
978 [BPF_ALU | BPF_XOR | BPF_K] = true,
979 [BPF_ALU | BPF_XOR | BPF_X] = true,
980 [BPF_ALU | BPF_LSH | BPF_K] = true,
981 [BPF_ALU | BPF_LSH | BPF_X] = true,
982 [BPF_ALU | BPF_RSH | BPF_K] = true,
983 [BPF_ALU | BPF_RSH | BPF_X] = true,
984 [BPF_ALU | BPF_NEG] = true,
985
986 [BPF_LD | BPF_W | BPF_ABS] = true,
987 [BPF_LD | BPF_H | BPF_ABS] = true,
988 [BPF_LD | BPF_B | BPF_ABS] = true,
989 [BPF_LD | BPF_W | BPF_LEN] = true,
990 [BPF_LD | BPF_W | BPF_IND] = true,
991 [BPF_LD | BPF_H | BPF_IND] = true,
992 [BPF_LD | BPF_B | BPF_IND] = true,
993 [BPF_LD | BPF_IMM] = true,
994 [BPF_LD | BPF_MEM] = true,
995 [BPF_LDX | BPF_W | BPF_LEN] = true,
996 [BPF_LDX | BPF_B | BPF_MSH] = true,
997 [BPF_LDX | BPF_IMM] = true,
998 [BPF_LDX | BPF_MEM] = true,
999
1000 [BPF_ST] = true,
1001 [BPF_STX] = true,
1002
1003 [BPF_MISC | BPF_TAX] = true,
1004 [BPF_MISC | BPF_TXA] = true,
1005
1006 [BPF_RET | BPF_K] = true,
1007 [BPF_RET | BPF_A] = true,
1008
1009 [BPF_JMP | BPF_JA] = true,
1010 [BPF_JMP | BPF_JEQ | BPF_K] = true,
1011 [BPF_JMP | BPF_JEQ | BPF_X] = true,
1012 [BPF_JMP | BPF_JGE | BPF_K] = true,
1013 [BPF_JMP | BPF_JGE | BPF_X] = true,
1014 [BPF_JMP | BPF_JGT | BPF_K] = true,
1015 [BPF_JMP | BPF_JGT | BPF_X] = true,
1016 [BPF_JMP | BPF_JSET | BPF_K] = true,
1017 [BPF_JMP | BPF_JSET | BPF_X] = true,
1018 };
1019
1020 if (code_to_probe >= ARRAY_SIZE(codes))
1021 return false;
1022
1023 return codes[code_to_probe];
1024}
1025
1026static bool bpf_check_basics_ok(const struct sock_filter *filter,
1027 unsigned int flen)
1028{
1029 if (filter == NULL)
1030 return false;
1031 if (flen == 0 || flen > BPF_MAXINSNS)
1032 return false;
1033
1034 return true;
1035}
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051static int bpf_check_classic(const struct sock_filter *filter,
1052 unsigned int flen)
1053{
1054 bool anc_found;
1055 int pc;
1056
1057
1058 for (pc = 0; pc < flen; pc++) {
1059 const struct sock_filter *ftest = &filter[pc];
1060
1061
1062 if (!chk_code_allowed(ftest->code))
1063 return -EINVAL;
1064
1065
1066 switch (ftest->code) {
1067 case BPF_ALU | BPF_DIV | BPF_K:
1068 case BPF_ALU | BPF_MOD | BPF_K:
1069
1070 if (ftest->k == 0)
1071 return -EINVAL;
1072 break;
1073 case BPF_ALU | BPF_LSH | BPF_K:
1074 case BPF_ALU | BPF_RSH | BPF_K:
1075 if (ftest->k >= 32)
1076 return -EINVAL;
1077 break;
1078 case BPF_LD | BPF_MEM:
1079 case BPF_LDX | BPF_MEM:
1080 case BPF_ST:
1081 case BPF_STX:
1082
1083 if (ftest->k >= BPF_MEMWORDS)
1084 return -EINVAL;
1085 break;
1086 case BPF_JMP | BPF_JA:
1087
1088
1089
1090
1091 if (ftest->k >= (unsigned int)(flen - pc - 1))
1092 return -EINVAL;
1093 break;
1094 case BPF_JMP | BPF_JEQ | BPF_K:
1095 case BPF_JMP | BPF_JEQ | BPF_X:
1096 case BPF_JMP | BPF_JGE | BPF_K:
1097 case BPF_JMP | BPF_JGE | BPF_X:
1098 case BPF_JMP | BPF_JGT | BPF_K:
1099 case BPF_JMP | BPF_JGT | BPF_X:
1100 case BPF_JMP | BPF_JSET | BPF_K:
1101 case BPF_JMP | BPF_JSET | BPF_X:
1102
1103 if (pc + ftest->jt + 1 >= flen ||
1104 pc + ftest->jf + 1 >= flen)
1105 return -EINVAL;
1106 break;
1107 case BPF_LD | BPF_W | BPF_ABS:
1108 case BPF_LD | BPF_H | BPF_ABS:
1109 case BPF_LD | BPF_B | BPF_ABS:
1110 anc_found = false;
1111 if (bpf_anc_helper(ftest) & BPF_ANC)
1112 anc_found = true;
1113
1114 if (anc_found == false && ftest->k >= SKF_AD_OFF)
1115 return -EINVAL;
1116 }
1117 }
1118
1119
1120 switch (filter[flen - 1].code) {
1121 case BPF_RET | BPF_K:
1122 case BPF_RET | BPF_A:
1123 return check_load_and_stores(filter, flen);
1124 }
1125
1126 return -EINVAL;
1127}
1128
1129static int bpf_prog_store_orig_filter(struct bpf_prog *fp,
1130 const struct sock_fprog *fprog)
1131{
1132 unsigned int fsize = bpf_classic_proglen(fprog);
1133 struct sock_fprog_kern *fkprog;
1134
1135 fp->orig_prog = kmalloc(sizeof(*fkprog), GFP_KERNEL);
1136 if (!fp->orig_prog)
1137 return -ENOMEM;
1138
1139 fkprog = fp->orig_prog;
1140 fkprog->len = fprog->len;
1141
1142 fkprog->filter = kmemdup(fp->insns, fsize,
1143 GFP_KERNEL | __GFP_NOWARN);
1144 if (!fkprog->filter) {
1145 kfree(fp->orig_prog);
1146 return -ENOMEM;
1147 }
1148
1149 return 0;
1150}
1151
1152static void bpf_release_orig_filter(struct bpf_prog *fp)
1153{
1154 struct sock_fprog_kern *fprog = fp->orig_prog;
1155
1156 if (fprog) {
1157 kfree(fprog->filter);
1158 kfree(fprog);
1159 }
1160}
1161
1162static void __bpf_prog_release(struct bpf_prog *prog)
1163{
1164 if (prog->type == BPF_PROG_TYPE_SOCKET_FILTER) {
1165 bpf_prog_put(prog);
1166 } else {
1167 bpf_release_orig_filter(prog);
1168 bpf_prog_free(prog);
1169 }
1170}
1171
1172static void __sk_filter_release(struct sk_filter *fp)
1173{
1174 __bpf_prog_release(fp->prog);
1175 kfree(fp);
1176}
1177
1178
1179
1180
1181
1182static void sk_filter_release_rcu(struct rcu_head *rcu)
1183{
1184 struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu);
1185
1186 __sk_filter_release(fp);
1187}
1188
1189
1190
1191
1192
1193
1194
1195static void sk_filter_release(struct sk_filter *fp)
1196{
1197 if (refcount_dec_and_test(&fp->refcnt))
1198 call_rcu(&fp->rcu, sk_filter_release_rcu);
1199}
1200
1201void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1202{
1203 u32 filter_size = bpf_prog_size(fp->prog->len);
1204
1205 atomic_sub(filter_size, &sk->sk_omem_alloc);
1206 sk_filter_release(fp);
1207}
1208
1209
1210
1211
1212static bool __sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1213{
1214 u32 filter_size = bpf_prog_size(fp->prog->len);
1215
1216
1217 if (filter_size <= sysctl_optmem_max &&
1218 atomic_read(&sk->sk_omem_alloc) + filter_size < sysctl_optmem_max) {
1219 atomic_add(filter_size, &sk->sk_omem_alloc);
1220 return true;
1221 }
1222 return false;
1223}
1224
1225bool sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1226{
1227 if (!refcount_inc_not_zero(&fp->refcnt))
1228 return false;
1229
1230 if (!__sk_filter_charge(sk, fp)) {
1231 sk_filter_release(fp);
1232 return false;
1233 }
1234 return true;
1235}
1236
1237static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp)
1238{
1239 struct sock_filter *old_prog;
1240 struct bpf_prog *old_fp;
1241 int err, new_len, old_len = fp->len;
1242 bool seen_ld_abs = false;
1243
1244
1245
1246
1247
1248
1249 BUILD_BUG_ON(sizeof(struct sock_filter) !=
1250 sizeof(struct bpf_insn));
1251
1252
1253
1254
1255
1256 old_prog = kmemdup(fp->insns, old_len * sizeof(struct sock_filter),
1257 GFP_KERNEL | __GFP_NOWARN);
1258 if (!old_prog) {
1259 err = -ENOMEM;
1260 goto out_err;
1261 }
1262
1263
1264 err = bpf_convert_filter(old_prog, old_len, NULL, &new_len,
1265 &seen_ld_abs);
1266 if (err)
1267 goto out_err_free;
1268
1269
1270 old_fp = fp;
1271 fp = bpf_prog_realloc(old_fp, bpf_prog_size(new_len), 0);
1272 if (!fp) {
1273
1274
1275
1276 fp = old_fp;
1277 err = -ENOMEM;
1278 goto out_err_free;
1279 }
1280
1281 fp->len = new_len;
1282
1283
1284 err = bpf_convert_filter(old_prog, old_len, fp, &new_len,
1285 &seen_ld_abs);
1286 if (err)
1287
1288
1289
1290
1291
1292 goto out_err_free;
1293
1294 fp = bpf_prog_select_runtime(fp, &err);
1295 if (err)
1296 goto out_err_free;
1297
1298 kfree(old_prog);
1299 return fp;
1300
1301out_err_free:
1302 kfree(old_prog);
1303out_err:
1304 __bpf_prog_release(fp);
1305 return ERR_PTR(err);
1306}
1307
1308static struct bpf_prog *bpf_prepare_filter(struct bpf_prog *fp,
1309 bpf_aux_classic_check_t trans)
1310{
1311 int err;
1312
1313 fp->bpf_func = NULL;
1314 fp->jited = 0;
1315
1316 err = bpf_check_classic(fp->insns, fp->len);
1317 if (err) {
1318 __bpf_prog_release(fp);
1319 return ERR_PTR(err);
1320 }
1321
1322
1323
1324
1325 if (trans) {
1326 err = trans(fp->insns, fp->len);
1327 if (err) {
1328 __bpf_prog_release(fp);
1329 return ERR_PTR(err);
1330 }
1331 }
1332
1333
1334
1335
1336 bpf_jit_compile(fp);
1337
1338
1339
1340
1341 if (!fp->jited)
1342 fp = bpf_migrate_filter(fp);
1343
1344 return fp;
1345}
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog)
1358{
1359 unsigned int fsize = bpf_classic_proglen(fprog);
1360 struct bpf_prog *fp;
1361
1362
1363 if (!bpf_check_basics_ok(fprog->filter, fprog->len))
1364 return -EINVAL;
1365
1366 fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
1367 if (!fp)
1368 return -ENOMEM;
1369
1370 memcpy(fp->insns, fprog->filter, fsize);
1371
1372 fp->len = fprog->len;
1373
1374
1375
1376
1377 fp->orig_prog = NULL;
1378
1379
1380
1381
1382 fp = bpf_prepare_filter(fp, NULL);
1383 if (IS_ERR(fp))
1384 return PTR_ERR(fp);
1385
1386 *pfp = fp;
1387 return 0;
1388}
1389EXPORT_SYMBOL_GPL(bpf_prog_create);
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
1403 bpf_aux_classic_check_t trans, bool save_orig)
1404{
1405 unsigned int fsize = bpf_classic_proglen(fprog);
1406 struct bpf_prog *fp;
1407 int err;
1408
1409
1410 if (!bpf_check_basics_ok(fprog->filter, fprog->len))
1411 return -EINVAL;
1412
1413 fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
1414 if (!fp)
1415 return -ENOMEM;
1416
1417 if (copy_from_user(fp->insns, fprog->filter, fsize)) {
1418 __bpf_prog_free(fp);
1419 return -EFAULT;
1420 }
1421
1422 fp->len = fprog->len;
1423 fp->orig_prog = NULL;
1424
1425 if (save_orig) {
1426 err = bpf_prog_store_orig_filter(fp, fprog);
1427 if (err) {
1428 __bpf_prog_free(fp);
1429 return -ENOMEM;
1430 }
1431 }
1432
1433
1434
1435
1436 fp = bpf_prepare_filter(fp, trans);
1437 if (IS_ERR(fp))
1438 return PTR_ERR(fp);
1439
1440 *pfp = fp;
1441 return 0;
1442}
1443EXPORT_SYMBOL_GPL(bpf_prog_create_from_user);
1444
1445void bpf_prog_destroy(struct bpf_prog *fp)
1446{
1447 __bpf_prog_release(fp);
1448}
1449EXPORT_SYMBOL_GPL(bpf_prog_destroy);
1450
1451static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk)
1452{
1453 struct sk_filter *fp, *old_fp;
1454
1455 fp = kmalloc(sizeof(*fp), GFP_KERNEL);
1456 if (!fp)
1457 return -ENOMEM;
1458
1459 fp->prog = prog;
1460
1461 if (!__sk_filter_charge(sk, fp)) {
1462 kfree(fp);
1463 return -ENOMEM;
1464 }
1465 refcount_set(&fp->refcnt, 1);
1466
1467 old_fp = rcu_dereference_protected(sk->sk_filter,
1468 lockdep_sock_is_held(sk));
1469 rcu_assign_pointer(sk->sk_filter, fp);
1470
1471 if (old_fp)
1472 sk_filter_uncharge(sk, old_fp);
1473
1474 return 0;
1475}
1476
1477static
1478struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk)
1479{
1480 unsigned int fsize = bpf_classic_proglen(fprog);
1481 struct bpf_prog *prog;
1482 int err;
1483
1484 if (sock_flag(sk, SOCK_FILTER_LOCKED))
1485 return ERR_PTR(-EPERM);
1486
1487
1488 if (!bpf_check_basics_ok(fprog->filter, fprog->len))
1489 return ERR_PTR(-EINVAL);
1490
1491 prog = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
1492 if (!prog)
1493 return ERR_PTR(-ENOMEM);
1494
1495 if (copy_from_user(prog->insns, fprog->filter, fsize)) {
1496 __bpf_prog_free(prog);
1497 return ERR_PTR(-EFAULT);
1498 }
1499
1500 prog->len = fprog->len;
1501
1502 err = bpf_prog_store_orig_filter(prog, fprog);
1503 if (err) {
1504 __bpf_prog_free(prog);
1505 return ERR_PTR(-ENOMEM);
1506 }
1507
1508
1509
1510
1511 return bpf_prepare_filter(prog, NULL);
1512}
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
1525{
1526 struct bpf_prog *prog = __get_filter(fprog, sk);
1527 int err;
1528
1529 if (IS_ERR(prog))
1530 return PTR_ERR(prog);
1531
1532 err = __sk_attach_prog(prog, sk);
1533 if (err < 0) {
1534 __bpf_prog_release(prog);
1535 return err;
1536 }
1537
1538 return 0;
1539}
1540EXPORT_SYMBOL_GPL(sk_attach_filter);
1541
1542int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk)
1543{
1544 struct bpf_prog *prog = __get_filter(fprog, sk);
1545 int err;
1546
1547 if (IS_ERR(prog))
1548 return PTR_ERR(prog);
1549
1550 if (bpf_prog_size(prog->len) > sysctl_optmem_max)
1551 err = -ENOMEM;
1552 else
1553 err = reuseport_attach_prog(sk, prog);
1554
1555 if (err)
1556 __bpf_prog_release(prog);
1557
1558 return err;
1559}
1560
1561static struct bpf_prog *__get_bpf(u32 ufd, struct sock *sk)
1562{
1563 if (sock_flag(sk, SOCK_FILTER_LOCKED))
1564 return ERR_PTR(-EPERM);
1565
1566 return bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER);
1567}
1568
1569int sk_attach_bpf(u32 ufd, struct sock *sk)
1570{
1571 struct bpf_prog *prog = __get_bpf(ufd, sk);
1572 int err;
1573
1574 if (IS_ERR(prog))
1575 return PTR_ERR(prog);
1576
1577 err = __sk_attach_prog(prog, sk);
1578 if (err < 0) {
1579 bpf_prog_put(prog);
1580 return err;
1581 }
1582
1583 return 0;
1584}
1585
1586int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk)
1587{
1588 struct bpf_prog *prog;
1589 int err;
1590
1591 if (sock_flag(sk, SOCK_FILTER_LOCKED))
1592 return -EPERM;
1593
1594 prog = bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER);
1595 if (PTR_ERR(prog) == -EINVAL)
1596 prog = bpf_prog_get_type(ufd, BPF_PROG_TYPE_SK_REUSEPORT);
1597 if (IS_ERR(prog))
1598 return PTR_ERR(prog);
1599
1600 if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) {
1601
1602
1603
1604
1605
1606 if ((sk->sk_type != SOCK_STREAM &&
1607 sk->sk_type != SOCK_DGRAM) ||
1608 (sk->sk_protocol != IPPROTO_UDP &&
1609 sk->sk_protocol != IPPROTO_TCP) ||
1610 (sk->sk_family != AF_INET &&
1611 sk->sk_family != AF_INET6)) {
1612 err = -ENOTSUPP;
1613 goto err_prog_put;
1614 }
1615 } else {
1616
1617 if (bpf_prog_size(prog->len) > sysctl_optmem_max) {
1618 err = -ENOMEM;
1619 goto err_prog_put;
1620 }
1621 }
1622
1623 err = reuseport_attach_prog(sk, prog);
1624err_prog_put:
1625 if (err)
1626 bpf_prog_put(prog);
1627
1628 return err;
1629}
1630
1631void sk_reuseport_prog_free(struct bpf_prog *prog)
1632{
1633 if (!prog)
1634 return;
1635
1636 if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
1637 bpf_prog_put(prog);
1638 else
1639 bpf_prog_destroy(prog);
1640}
1641
1642struct bpf_scratchpad {
1643 union {
1644 __be32 diff[MAX_BPF_STACK / sizeof(__be32)];
1645 u8 buff[MAX_BPF_STACK];
1646 };
1647};
1648
1649static DEFINE_PER_CPU(struct bpf_scratchpad, bpf_sp);
1650
1651static inline int __bpf_try_make_writable(struct sk_buff *skb,
1652 unsigned int write_len)
1653{
1654 return skb_ensure_writable(skb, write_len);
1655}
1656
1657static inline int bpf_try_make_writable(struct sk_buff *skb,
1658 unsigned int write_len)
1659{
1660 int err = __bpf_try_make_writable(skb, write_len);
1661
1662 bpf_compute_data_pointers(skb);
1663 return err;
1664}
1665
1666static int bpf_try_make_head_writable(struct sk_buff *skb)
1667{
1668 return bpf_try_make_writable(skb, skb_headlen(skb));
1669}
1670
1671static inline void bpf_push_mac_rcsum(struct sk_buff *skb)
1672{
1673 if (skb_at_tc_ingress(skb))
1674 skb_postpush_rcsum(skb, skb_mac_header(skb), skb->mac_len);
1675}
1676
1677static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
1678{
1679 if (skb_at_tc_ingress(skb))
1680 skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
1681}
1682
1683BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
1684 const void *, from, u32, len, u64, flags)
1685{
1686 void *ptr;
1687
1688 if (unlikely(flags & ~(BPF_F_RECOMPUTE_CSUM | BPF_F_INVALIDATE_HASH)))
1689 return -EINVAL;
1690 if (unlikely(offset > 0xffff))
1691 return -EFAULT;
1692 if (unlikely(bpf_try_make_writable(skb, offset + len)))
1693 return -EFAULT;
1694
1695 ptr = skb->data + offset;
1696 if (flags & BPF_F_RECOMPUTE_CSUM)
1697 __skb_postpull_rcsum(skb, ptr, len, offset);
1698
1699 memcpy(ptr, from, len);
1700
1701 if (flags & BPF_F_RECOMPUTE_CSUM)
1702 __skb_postpush_rcsum(skb, ptr, len, offset);
1703 if (flags & BPF_F_INVALIDATE_HASH)
1704 skb_clear_hash(skb);
1705
1706 return 0;
1707}
1708
1709static const struct bpf_func_proto bpf_skb_store_bytes_proto = {
1710 .func = bpf_skb_store_bytes,
1711 .gpl_only = false,
1712 .ret_type = RET_INTEGER,
1713 .arg1_type = ARG_PTR_TO_CTX,
1714 .arg2_type = ARG_ANYTHING,
1715 .arg3_type = ARG_PTR_TO_MEM,
1716 .arg4_type = ARG_CONST_SIZE,
1717 .arg5_type = ARG_ANYTHING,
1718};
1719
1720BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
1721 void *, to, u32, len)
1722{
1723 void *ptr;
1724
1725 if (unlikely(offset > 0xffff))
1726 goto err_clear;
1727
1728 ptr = skb_header_pointer(skb, offset, len, to);
1729 if (unlikely(!ptr))
1730 goto err_clear;
1731 if (ptr != to)
1732 memcpy(to, ptr, len);
1733
1734 return 0;
1735err_clear:
1736 memset(to, 0, len);
1737 return -EFAULT;
1738}
1739
1740static const struct bpf_func_proto bpf_skb_load_bytes_proto = {
1741 .func = bpf_skb_load_bytes,
1742 .gpl_only = false,
1743 .ret_type = RET_INTEGER,
1744 .arg1_type = ARG_PTR_TO_CTX,
1745 .arg2_type = ARG_ANYTHING,
1746 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
1747 .arg4_type = ARG_CONST_SIZE,
1748};
1749
1750BPF_CALL_4(bpf_flow_dissector_load_bytes,
1751 const struct bpf_flow_dissector *, ctx, u32, offset,
1752 void *, to, u32, len)
1753{
1754 void *ptr;
1755
1756 if (unlikely(offset > 0xffff))
1757 goto err_clear;
1758
1759 if (unlikely(!ctx->skb))
1760 goto err_clear;
1761
1762 ptr = skb_header_pointer(ctx->skb, offset, len, to);
1763 if (unlikely(!ptr))
1764 goto err_clear;
1765 if (ptr != to)
1766 memcpy(to, ptr, len);
1767
1768 return 0;
1769err_clear:
1770 memset(to, 0, len);
1771 return -EFAULT;
1772}
1773
1774static const struct bpf_func_proto bpf_flow_dissector_load_bytes_proto = {
1775 .func = bpf_flow_dissector_load_bytes,
1776 .gpl_only = false,
1777 .ret_type = RET_INTEGER,
1778 .arg1_type = ARG_PTR_TO_CTX,
1779 .arg2_type = ARG_ANYTHING,
1780 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
1781 .arg4_type = ARG_CONST_SIZE,
1782};
1783
1784BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb,
1785 u32, offset, void *, to, u32, len, u32, start_header)
1786{
1787 u8 *end = skb_tail_pointer(skb);
1788 u8 *start, *ptr;
1789
1790 if (unlikely(offset > 0xffff))
1791 goto err_clear;
1792
1793 switch (start_header) {
1794 case BPF_HDR_START_MAC:
1795 if (unlikely(!skb_mac_header_was_set(skb)))
1796 goto err_clear;
1797 start = skb_mac_header(skb);
1798 break;
1799 case BPF_HDR_START_NET:
1800 start = skb_network_header(skb);
1801 break;
1802 default:
1803 goto err_clear;
1804 }
1805
1806 ptr = start + offset;
1807
1808 if (likely(ptr + len <= end)) {
1809 memcpy(to, ptr, len);
1810 return 0;
1811 }
1812
1813err_clear:
1814 memset(to, 0, len);
1815 return -EFAULT;
1816}
1817
1818static const struct bpf_func_proto bpf_skb_load_bytes_relative_proto = {
1819 .func = bpf_skb_load_bytes_relative,
1820 .gpl_only = false,
1821 .ret_type = RET_INTEGER,
1822 .arg1_type = ARG_PTR_TO_CTX,
1823 .arg2_type = ARG_ANYTHING,
1824 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
1825 .arg4_type = ARG_CONST_SIZE,
1826 .arg5_type = ARG_ANYTHING,
1827};
1828
1829BPF_CALL_2(bpf_skb_pull_data, struct sk_buff *, skb, u32, len)
1830{
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840 return bpf_try_make_writable(skb, len ? : skb_headlen(skb));
1841}
1842
1843static const struct bpf_func_proto bpf_skb_pull_data_proto = {
1844 .func = bpf_skb_pull_data,
1845 .gpl_only = false,
1846 .ret_type = RET_INTEGER,
1847 .arg1_type = ARG_PTR_TO_CTX,
1848 .arg2_type = ARG_ANYTHING,
1849};
1850
1851BPF_CALL_1(bpf_sk_fullsock, struct sock *, sk)
1852{
1853 return sk_fullsock(sk) ? (unsigned long)sk : (unsigned long)NULL;
1854}
1855
1856static const struct bpf_func_proto bpf_sk_fullsock_proto = {
1857 .func = bpf_sk_fullsock,
1858 .gpl_only = false,
1859 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
1860 .arg1_type = ARG_PTR_TO_SOCK_COMMON,
1861};
1862
1863static inline int sk_skb_try_make_writable(struct sk_buff *skb,
1864 unsigned int write_len)
1865{
1866 return __bpf_try_make_writable(skb, write_len);
1867}
1868
1869BPF_CALL_2(sk_skb_pull_data, struct sk_buff *, skb, u32, len)
1870{
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880 return sk_skb_try_make_writable(skb, len ? : skb_headlen(skb));
1881}
1882
1883static const struct bpf_func_proto sk_skb_pull_data_proto = {
1884 .func = sk_skb_pull_data,
1885 .gpl_only = false,
1886 .ret_type = RET_INTEGER,
1887 .arg1_type = ARG_PTR_TO_CTX,
1888 .arg2_type = ARG_ANYTHING,
1889};
1890
1891BPF_CALL_5(bpf_l3_csum_replace, struct sk_buff *, skb, u32, offset,
1892 u64, from, u64, to, u64, flags)
1893{
1894 __sum16 *ptr;
1895
1896 if (unlikely(flags & ~(BPF_F_HDR_FIELD_MASK)))
1897 return -EINVAL;
1898 if (unlikely(offset > 0xffff || offset & 1))
1899 return -EFAULT;
1900 if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr))))
1901 return -EFAULT;
1902
1903 ptr = (__sum16 *)(skb->data + offset);
1904 switch (flags & BPF_F_HDR_FIELD_MASK) {
1905 case 0:
1906 if (unlikely(from != 0))
1907 return -EINVAL;
1908
1909 csum_replace_by_diff(ptr, to);
1910 break;
1911 case 2:
1912 csum_replace2(ptr, from, to);
1913 break;
1914 case 4:
1915 csum_replace4(ptr, from, to);
1916 break;
1917 default:
1918 return -EINVAL;
1919 }
1920
1921 return 0;
1922}
1923
1924static const struct bpf_func_proto bpf_l3_csum_replace_proto = {
1925 .func = bpf_l3_csum_replace,
1926 .gpl_only = false,
1927 .ret_type = RET_INTEGER,
1928 .arg1_type = ARG_PTR_TO_CTX,
1929 .arg2_type = ARG_ANYTHING,
1930 .arg3_type = ARG_ANYTHING,
1931 .arg4_type = ARG_ANYTHING,
1932 .arg5_type = ARG_ANYTHING,
1933};
1934
1935BPF_CALL_5(bpf_l4_csum_replace, struct sk_buff *, skb, u32, offset,
1936 u64, from, u64, to, u64, flags)
1937{
1938 bool is_pseudo = flags & BPF_F_PSEUDO_HDR;
1939 bool is_mmzero = flags & BPF_F_MARK_MANGLED_0;
1940 bool do_mforce = flags & BPF_F_MARK_ENFORCE;
1941 __sum16 *ptr;
1942
1943 if (unlikely(flags & ~(BPF_F_MARK_MANGLED_0 | BPF_F_MARK_ENFORCE |
1944 BPF_F_PSEUDO_HDR | BPF_F_HDR_FIELD_MASK)))
1945 return -EINVAL;
1946 if (unlikely(offset > 0xffff || offset & 1))
1947 return -EFAULT;
1948 if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr))))
1949 return -EFAULT;
1950
1951 ptr = (__sum16 *)(skb->data + offset);
1952 if (is_mmzero && !do_mforce && !*ptr)
1953 return 0;
1954
1955 switch (flags & BPF_F_HDR_FIELD_MASK) {
1956 case 0:
1957 if (unlikely(from != 0))
1958 return -EINVAL;
1959
1960 inet_proto_csum_replace_by_diff(ptr, skb, to, is_pseudo);
1961 break;
1962 case 2:
1963 inet_proto_csum_replace2(ptr, skb, from, to, is_pseudo);
1964 break;
1965 case 4:
1966 inet_proto_csum_replace4(ptr, skb, from, to, is_pseudo);
1967 break;
1968 default:
1969 return -EINVAL;
1970 }
1971
1972 if (is_mmzero && !*ptr)
1973 *ptr = CSUM_MANGLED_0;
1974 return 0;
1975}
1976
1977static const struct bpf_func_proto bpf_l4_csum_replace_proto = {
1978 .func = bpf_l4_csum_replace,
1979 .gpl_only = false,
1980 .ret_type = RET_INTEGER,
1981 .arg1_type = ARG_PTR_TO_CTX,
1982 .arg2_type = ARG_ANYTHING,
1983 .arg3_type = ARG_ANYTHING,
1984 .arg4_type = ARG_ANYTHING,
1985 .arg5_type = ARG_ANYTHING,
1986};
1987
1988BPF_CALL_5(bpf_csum_diff, __be32 *, from, u32, from_size,
1989 __be32 *, to, u32, to_size, __wsum, seed)
1990{
1991 struct bpf_scratchpad *sp = this_cpu_ptr(&bpf_sp);
1992 u32 diff_size = from_size + to_size;
1993 int i, j = 0;
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003 if (unlikely(((from_size | to_size) & (sizeof(__be32) - 1)) ||
2004 diff_size > sizeof(sp->diff)))
2005 return -EINVAL;
2006
2007 for (i = 0; i < from_size / sizeof(__be32); i++, j++)
2008 sp->diff[j] = ~from[i];
2009 for (i = 0; i < to_size / sizeof(__be32); i++, j++)
2010 sp->diff[j] = to[i];
2011
2012 return csum_partial(sp->diff, diff_size, seed);
2013}
2014
2015static const struct bpf_func_proto bpf_csum_diff_proto = {
2016 .func = bpf_csum_diff,
2017 .gpl_only = false,
2018 .pkt_access = true,
2019 .ret_type = RET_INTEGER,
2020 .arg1_type = ARG_PTR_TO_MEM_OR_NULL,
2021 .arg2_type = ARG_CONST_SIZE_OR_ZERO,
2022 .arg3_type = ARG_PTR_TO_MEM_OR_NULL,
2023 .arg4_type = ARG_CONST_SIZE_OR_ZERO,
2024 .arg5_type = ARG_ANYTHING,
2025};
2026
2027BPF_CALL_2(bpf_csum_update, struct sk_buff *, skb, __wsum, csum)
2028{
2029
2030
2031
2032
2033 if (skb->ip_summed == CHECKSUM_COMPLETE)
2034 return (skb->csum = csum_add(skb->csum, csum));
2035
2036 return -ENOTSUPP;
2037}
2038
2039static const struct bpf_func_proto bpf_csum_update_proto = {
2040 .func = bpf_csum_update,
2041 .gpl_only = false,
2042 .ret_type = RET_INTEGER,
2043 .arg1_type = ARG_PTR_TO_CTX,
2044 .arg2_type = ARG_ANYTHING,
2045};
2046
2047BPF_CALL_2(bpf_csum_level, struct sk_buff *, skb, u64, level)
2048{
2049
2050
2051
2052
2053 switch (level) {
2054 case BPF_CSUM_LEVEL_INC:
2055 __skb_incr_checksum_unnecessary(skb);
2056 break;
2057 case BPF_CSUM_LEVEL_DEC:
2058 __skb_decr_checksum_unnecessary(skb);
2059 break;
2060 case BPF_CSUM_LEVEL_RESET:
2061 __skb_reset_checksum_unnecessary(skb);
2062 break;
2063 case BPF_CSUM_LEVEL_QUERY:
2064 return skb->ip_summed == CHECKSUM_UNNECESSARY ?
2065 skb->csum_level : -EACCES;
2066 default:
2067 return -EINVAL;
2068 }
2069
2070 return 0;
2071}
2072
2073static const struct bpf_func_proto bpf_csum_level_proto = {
2074 .func = bpf_csum_level,
2075 .gpl_only = false,
2076 .ret_type = RET_INTEGER,
2077 .arg1_type = ARG_PTR_TO_CTX,
2078 .arg2_type = ARG_ANYTHING,
2079};
2080
2081static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb)
2082{
2083 return dev_forward_skb_nomtu(dev, skb);
2084}
2085
2086static inline int __bpf_rx_skb_no_mac(struct net_device *dev,
2087 struct sk_buff *skb)
2088{
2089 int ret = ____dev_forward_skb(dev, skb, false);
2090
2091 if (likely(!ret)) {
2092 skb->dev = dev;
2093 ret = netif_rx(skb);
2094 }
2095
2096 return ret;
2097}
2098
2099static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb)
2100{
2101 int ret;
2102
2103 if (dev_xmit_recursion()) {
2104 net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
2105 kfree_skb(skb);
2106 return -ENETDOWN;
2107 }
2108
2109 skb->dev = dev;
2110 skb->tstamp = 0;
2111
2112 dev_xmit_recursion_inc();
2113 ret = dev_queue_xmit(skb);
2114 dev_xmit_recursion_dec();
2115
2116 return ret;
2117}
2118
2119static int __bpf_redirect_no_mac(struct sk_buff *skb, struct net_device *dev,
2120 u32 flags)
2121{
2122 unsigned int mlen = skb_network_offset(skb);
2123
2124 if (mlen) {
2125 __skb_pull(skb, mlen);
2126
2127
2128
2129
2130
2131
2132 if (!skb_at_tc_ingress(skb))
2133 skb_postpull_rcsum(skb, skb_mac_header(skb), mlen);
2134 }
2135 skb_pop_mac_header(skb);
2136 skb_reset_mac_len(skb);
2137 return flags & BPF_F_INGRESS ?
2138 __bpf_rx_skb_no_mac(dev, skb) : __bpf_tx_skb(dev, skb);
2139}
2140
2141static int __bpf_redirect_common(struct sk_buff *skb, struct net_device *dev,
2142 u32 flags)
2143{
2144
2145 if (unlikely(skb->mac_header >= skb->network_header)) {
2146 kfree_skb(skb);
2147 return -ERANGE;
2148 }
2149
2150 bpf_push_mac_rcsum(skb);
2151 return flags & BPF_F_INGRESS ?
2152 __bpf_rx_skb(dev, skb) : __bpf_tx_skb(dev, skb);
2153}
2154
2155static int __bpf_redirect(struct sk_buff *skb, struct net_device *dev,
2156 u32 flags)
2157{
2158 if (dev_is_mac_header_xmit(dev))
2159 return __bpf_redirect_common(skb, dev, flags);
2160 else
2161 return __bpf_redirect_no_mac(skb, dev, flags);
2162}
2163
2164#if IS_ENABLED(CONFIG_IPV6)
2165static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb,
2166 struct net_device *dev, struct bpf_nh_params *nh)
2167{
2168 u32 hh_len = LL_RESERVED_SPACE(dev);
2169 const struct in6_addr *nexthop;
2170 struct dst_entry *dst = NULL;
2171 struct neighbour *neigh;
2172
2173 if (dev_xmit_recursion()) {
2174 net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
2175 goto out_drop;
2176 }
2177
2178 skb->dev = dev;
2179 skb->tstamp = 0;
2180
2181 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
2182 struct sk_buff *skb2;
2183
2184 skb2 = skb_realloc_headroom(skb, hh_len);
2185 if (unlikely(!skb2)) {
2186 kfree_skb(skb);
2187 return -ENOMEM;
2188 }
2189 if (skb->sk)
2190 skb_set_owner_w(skb2, skb->sk);
2191 consume_skb(skb);
2192 skb = skb2;
2193 }
2194
2195 rcu_read_lock_bh();
2196 if (!nh) {
2197 dst = skb_dst(skb);
2198 nexthop = rt6_nexthop(container_of(dst, struct rt6_info, dst),
2199 &ipv6_hdr(skb)->daddr);
2200 } else {
2201 nexthop = &nh->ipv6_nh;
2202 }
2203 neigh = ip_neigh_gw6(dev, nexthop);
2204 if (likely(!IS_ERR(neigh))) {
2205 int ret;
2206
2207 sock_confirm_neigh(skb, neigh);
2208 dev_xmit_recursion_inc();
2209 ret = neigh_output(neigh, skb, false);
2210 dev_xmit_recursion_dec();
2211 rcu_read_unlock_bh();
2212 return ret;
2213 }
2214 rcu_read_unlock_bh();
2215 if (dst)
2216 IP6_INC_STATS(dev_net(dst->dev),
2217 ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
2218out_drop:
2219 kfree_skb(skb);
2220 return -ENETDOWN;
2221}
2222
2223static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev,
2224 struct bpf_nh_params *nh)
2225{
2226 const struct ipv6hdr *ip6h = ipv6_hdr(skb);
2227 struct net *net = dev_net(dev);
2228 int err, ret = NET_XMIT_DROP;
2229
2230 if (!nh) {
2231 struct dst_entry *dst;
2232 struct flowi6 fl6 = {
2233 .flowi6_flags = FLOWI_FLAG_ANYSRC,
2234 .flowi6_mark = skb->mark,
2235 .flowlabel = ip6_flowinfo(ip6h),
2236 .flowi6_oif = dev->ifindex,
2237 .flowi6_proto = ip6h->nexthdr,
2238 .daddr = ip6h->daddr,
2239 .saddr = ip6h->saddr,
2240 };
2241
2242 dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL);
2243 if (IS_ERR(dst))
2244 goto out_drop;
2245
2246 skb_dst_set(skb, dst);
2247 } else if (nh->nh_family != AF_INET6) {
2248 goto out_drop;
2249 }
2250
2251 err = bpf_out_neigh_v6(net, skb, dev, nh);
2252 if (unlikely(net_xmit_eval(err)))
2253 dev->stats.tx_errors++;
2254 else
2255 ret = NET_XMIT_SUCCESS;
2256 goto out_xmit;
2257out_drop:
2258 dev->stats.tx_errors++;
2259 kfree_skb(skb);
2260out_xmit:
2261 return ret;
2262}
2263#else
2264static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev,
2265 struct bpf_nh_params *nh)
2266{
2267 kfree_skb(skb);
2268 return NET_XMIT_DROP;
2269}
2270#endif
2271
2272#if IS_ENABLED(CONFIG_INET)
2273static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb,
2274 struct net_device *dev, struct bpf_nh_params *nh)
2275{
2276 u32 hh_len = LL_RESERVED_SPACE(dev);
2277 struct neighbour *neigh;
2278 bool is_v6gw = false;
2279
2280 if (dev_xmit_recursion()) {
2281 net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
2282 goto out_drop;
2283 }
2284
2285 skb->dev = dev;
2286 skb->tstamp = 0;
2287
2288 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
2289 struct sk_buff *skb2;
2290
2291 skb2 = skb_realloc_headroom(skb, hh_len);
2292 if (unlikely(!skb2)) {
2293 kfree_skb(skb);
2294 return -ENOMEM;
2295 }
2296 if (skb->sk)
2297 skb_set_owner_w(skb2, skb->sk);
2298 consume_skb(skb);
2299 skb = skb2;
2300 }
2301
2302 rcu_read_lock_bh();
2303 if (!nh) {
2304 struct dst_entry *dst = skb_dst(skb);
2305 struct rtable *rt = container_of(dst, struct rtable, dst);
2306
2307 neigh = ip_neigh_for_gw(rt, skb, &is_v6gw);
2308 } else if (nh->nh_family == AF_INET6) {
2309 neigh = ip_neigh_gw6(dev, &nh->ipv6_nh);
2310 is_v6gw = true;
2311 } else if (nh->nh_family == AF_INET) {
2312 neigh = ip_neigh_gw4(dev, nh->ipv4_nh);
2313 } else {
2314 rcu_read_unlock_bh();
2315 goto out_drop;
2316 }
2317
2318 if (likely(!IS_ERR(neigh))) {
2319 int ret;
2320
2321 sock_confirm_neigh(skb, neigh);
2322 dev_xmit_recursion_inc();
2323 ret = neigh_output(neigh, skb, is_v6gw);
2324 dev_xmit_recursion_dec();
2325 rcu_read_unlock_bh();
2326 return ret;
2327 }
2328 rcu_read_unlock_bh();
2329out_drop:
2330 kfree_skb(skb);
2331 return -ENETDOWN;
2332}
2333
2334static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev,
2335 struct bpf_nh_params *nh)
2336{
2337 const struct iphdr *ip4h = ip_hdr(skb);
2338 struct net *net = dev_net(dev);
2339 int err, ret = NET_XMIT_DROP;
2340
2341 if (!nh) {
2342 struct flowi4 fl4 = {
2343 .flowi4_flags = FLOWI_FLAG_ANYSRC,
2344 .flowi4_mark = skb->mark,
2345 .flowi4_tos = RT_TOS(ip4h->tos),
2346 .flowi4_oif = dev->ifindex,
2347 .flowi4_proto = ip4h->protocol,
2348 .daddr = ip4h->daddr,
2349 .saddr = ip4h->saddr,
2350 };
2351 struct rtable *rt;
2352
2353 rt = ip_route_output_flow(net, &fl4, NULL);
2354 if (IS_ERR(rt))
2355 goto out_drop;
2356 if (rt->rt_type != RTN_UNICAST && rt->rt_type != RTN_LOCAL) {
2357 ip_rt_put(rt);
2358 goto out_drop;
2359 }
2360
2361 skb_dst_set(skb, &rt->dst);
2362 }
2363
2364 err = bpf_out_neigh_v4(net, skb, dev, nh);
2365 if (unlikely(net_xmit_eval(err)))
2366 dev->stats.tx_errors++;
2367 else
2368 ret = NET_XMIT_SUCCESS;
2369 goto out_xmit;
2370out_drop:
2371 dev->stats.tx_errors++;
2372 kfree_skb(skb);
2373out_xmit:
2374 return ret;
2375}
2376#else
2377static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev,
2378 struct bpf_nh_params *nh)
2379{
2380 kfree_skb(skb);
2381 return NET_XMIT_DROP;
2382}
2383#endif
2384
2385static int __bpf_redirect_neigh(struct sk_buff *skb, struct net_device *dev,
2386 struct bpf_nh_params *nh)
2387{
2388 struct ethhdr *ethh = eth_hdr(skb);
2389
2390 if (unlikely(skb->mac_header >= skb->network_header))
2391 goto out;
2392 bpf_push_mac_rcsum(skb);
2393 if (is_multicast_ether_addr(ethh->h_dest))
2394 goto out;
2395
2396 skb_pull(skb, sizeof(*ethh));
2397 skb_unset_mac_header(skb);
2398 skb_reset_network_header(skb);
2399
2400 if (skb->protocol == htons(ETH_P_IP))
2401 return __bpf_redirect_neigh_v4(skb, dev, nh);
2402 else if (skb->protocol == htons(ETH_P_IPV6))
2403 return __bpf_redirect_neigh_v6(skb, dev, nh);
2404out:
2405 kfree_skb(skb);
2406 return -ENOTSUPP;
2407}
2408
2409
2410enum {
2411 BPF_F_NEIGH = (1ULL << 1),
2412 BPF_F_PEER = (1ULL << 2),
2413 BPF_F_NEXTHOP = (1ULL << 3),
2414#define BPF_F_REDIRECT_INTERNAL (BPF_F_NEIGH | BPF_F_PEER | BPF_F_NEXTHOP)
2415};
2416
2417BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags)
2418{
2419 struct net_device *dev;
2420 struct sk_buff *clone;
2421 int ret;
2422
2423 if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))
2424 return -EINVAL;
2425
2426 dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex);
2427 if (unlikely(!dev))
2428 return -EINVAL;
2429
2430 clone = skb_clone(skb, GFP_ATOMIC);
2431 if (unlikely(!clone))
2432 return -ENOMEM;
2433
2434
2435
2436
2437
2438
2439 ret = bpf_try_make_head_writable(skb);
2440 if (unlikely(ret)) {
2441 kfree_skb(clone);
2442 return -ENOMEM;
2443 }
2444
2445 return __bpf_redirect(clone, dev, flags);
2446}
2447
2448static const struct bpf_func_proto bpf_clone_redirect_proto = {
2449 .func = bpf_clone_redirect,
2450 .gpl_only = false,
2451 .ret_type = RET_INTEGER,
2452 .arg1_type = ARG_PTR_TO_CTX,
2453 .arg2_type = ARG_ANYTHING,
2454 .arg3_type = ARG_ANYTHING,
2455};
2456
2457DEFINE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info);
2458EXPORT_PER_CPU_SYMBOL_GPL(bpf_redirect_info);
2459
2460int skb_do_redirect(struct sk_buff *skb)
2461{
2462 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
2463 struct net *net = dev_net(skb->dev);
2464 struct net_device *dev;
2465 u32 flags = ri->flags;
2466
2467 dev = dev_get_by_index_rcu(net, ri->tgt_index);
2468 ri->tgt_index = 0;
2469 ri->flags = 0;
2470 if (unlikely(!dev))
2471 goto out_drop;
2472 if (flags & BPF_F_PEER) {
2473 const struct net_device_ops *ops = dev->netdev_ops;
2474
2475 if (unlikely(!ops->ndo_get_peer_dev ||
2476 !skb_at_tc_ingress(skb)))
2477 goto out_drop;
2478 dev = ops->ndo_get_peer_dev(dev);
2479 if (unlikely(!dev ||
2480 !(dev->flags & IFF_UP) ||
2481 net_eq(net, dev_net(dev))))
2482 goto out_drop;
2483 skb->dev = dev;
2484 return -EAGAIN;
2485 }
2486 return flags & BPF_F_NEIGH ?
2487 __bpf_redirect_neigh(skb, dev, flags & BPF_F_NEXTHOP ?
2488 &ri->nh : NULL) :
2489 __bpf_redirect(skb, dev, flags);
2490out_drop:
2491 kfree_skb(skb);
2492 return -EINVAL;
2493}
2494
2495BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)
2496{
2497 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
2498
2499 if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))
2500 return TC_ACT_SHOT;
2501
2502 ri->flags = flags;
2503 ri->tgt_index = ifindex;
2504
2505 return TC_ACT_REDIRECT;
2506}
2507
2508static const struct bpf_func_proto bpf_redirect_proto = {
2509 .func = bpf_redirect,
2510 .gpl_only = false,
2511 .ret_type = RET_INTEGER,
2512 .arg1_type = ARG_ANYTHING,
2513 .arg2_type = ARG_ANYTHING,
2514};
2515
2516BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags)
2517{
2518 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
2519
2520 if (unlikely(flags))
2521 return TC_ACT_SHOT;
2522
2523 ri->flags = BPF_F_PEER;
2524 ri->tgt_index = ifindex;
2525
2526 return TC_ACT_REDIRECT;
2527}
2528
2529static const struct bpf_func_proto bpf_redirect_peer_proto = {
2530 .func = bpf_redirect_peer,
2531 .gpl_only = false,
2532 .ret_type = RET_INTEGER,
2533 .arg1_type = ARG_ANYTHING,
2534 .arg2_type = ARG_ANYTHING,
2535};
2536
2537BPF_CALL_4(bpf_redirect_neigh, u32, ifindex, struct bpf_redir_neigh *, params,
2538 int, plen, u64, flags)
2539{
2540 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
2541
2542 if (unlikely((plen && plen < sizeof(*params)) || flags))
2543 return TC_ACT_SHOT;
2544
2545 ri->flags = BPF_F_NEIGH | (plen ? BPF_F_NEXTHOP : 0);
2546 ri->tgt_index = ifindex;
2547
2548 BUILD_BUG_ON(sizeof(struct bpf_redir_neigh) != sizeof(struct bpf_nh_params));
2549 if (plen)
2550 memcpy(&ri->nh, params, sizeof(ri->nh));
2551
2552 return TC_ACT_REDIRECT;
2553}
2554
2555static const struct bpf_func_proto bpf_redirect_neigh_proto = {
2556 .func = bpf_redirect_neigh,
2557 .gpl_only = false,
2558 .ret_type = RET_INTEGER,
2559 .arg1_type = ARG_ANYTHING,
2560 .arg2_type = ARG_PTR_TO_MEM_OR_NULL,
2561 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
2562 .arg4_type = ARG_ANYTHING,
2563};
2564
2565BPF_CALL_2(bpf_msg_apply_bytes, struct sk_msg *, msg, u32, bytes)
2566{
2567 msg->apply_bytes = bytes;
2568 return 0;
2569}
2570
2571static const struct bpf_func_proto bpf_msg_apply_bytes_proto = {
2572 .func = bpf_msg_apply_bytes,
2573 .gpl_only = false,
2574 .ret_type = RET_INTEGER,
2575 .arg1_type = ARG_PTR_TO_CTX,
2576 .arg2_type = ARG_ANYTHING,
2577};
2578
2579BPF_CALL_2(bpf_msg_cork_bytes, struct sk_msg *, msg, u32, bytes)
2580{
2581 msg->cork_bytes = bytes;
2582 return 0;
2583}
2584
2585static const struct bpf_func_proto bpf_msg_cork_bytes_proto = {
2586 .func = bpf_msg_cork_bytes,
2587 .gpl_only = false,
2588 .ret_type = RET_INTEGER,
2589 .arg1_type = ARG_PTR_TO_CTX,
2590 .arg2_type = ARG_ANYTHING,
2591};
2592
2593BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start,
2594 u32, end, u64, flags)
2595{
2596 u32 len = 0, offset = 0, copy = 0, poffset = 0, bytes = end - start;
2597 u32 first_sge, last_sge, i, shift, bytes_sg_total;
2598 struct scatterlist *sge;
2599 u8 *raw, *to, *from;
2600 struct page *page;
2601
2602 if (unlikely(flags || end <= start))
2603 return -EINVAL;
2604
2605
2606 i = msg->sg.start;
2607 do {
2608 offset += len;
2609 len = sk_msg_elem(msg, i)->length;
2610 if (start < offset + len)
2611 break;
2612 sk_msg_iter_var_next(i);
2613 } while (i != msg->sg.end);
2614
2615 if (unlikely(start >= offset + len))
2616 return -EINVAL;
2617
2618 first_sge = i;
2619
2620
2621
2622 bytes_sg_total = start - offset + bytes;
2623 if (!test_bit(i, &msg->sg.copy) && bytes_sg_total <= len)
2624 goto out;
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636 do {
2637 copy += sk_msg_elem(msg, i)->length;
2638 sk_msg_iter_var_next(i);
2639 if (bytes_sg_total <= copy)
2640 break;
2641 } while (i != msg->sg.end);
2642 last_sge = i;
2643
2644 if (unlikely(bytes_sg_total > copy))
2645 return -EINVAL;
2646
2647 page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP,
2648 get_order(copy));
2649 if (unlikely(!page))
2650 return -ENOMEM;
2651
2652 raw = page_address(page);
2653 i = first_sge;
2654 do {
2655 sge = sk_msg_elem(msg, i);
2656 from = sg_virt(sge);
2657 len = sge->length;
2658 to = raw + poffset;
2659
2660 memcpy(to, from, len);
2661 poffset += len;
2662 sge->length = 0;
2663 put_page(sg_page(sge));
2664
2665 sk_msg_iter_var_next(i);
2666 } while (i != last_sge);
2667
2668 sg_set_page(&msg->sg.data[first_sge], page, copy, 0);
2669
2670
2671
2672
2673
2674 WARN_ON_ONCE(last_sge == first_sge);
2675 shift = last_sge > first_sge ?
2676 last_sge - first_sge - 1 :
2677 NR_MSG_FRAG_IDS - first_sge + last_sge - 1;
2678 if (!shift)
2679 goto out;
2680
2681 i = first_sge;
2682 sk_msg_iter_var_next(i);
2683 do {
2684 u32 move_from;
2685
2686 if (i + shift >= NR_MSG_FRAG_IDS)
2687 move_from = i + shift - NR_MSG_FRAG_IDS;
2688 else
2689 move_from = i + shift;
2690 if (move_from == msg->sg.end)
2691 break;
2692
2693 msg->sg.data[i] = msg->sg.data[move_from];
2694 msg->sg.data[move_from].length = 0;
2695 msg->sg.data[move_from].page_link = 0;
2696 msg->sg.data[move_from].offset = 0;
2697 sk_msg_iter_var_next(i);
2698 } while (1);
2699
2700 msg->sg.end = msg->sg.end - shift > msg->sg.end ?
2701 msg->sg.end - shift + NR_MSG_FRAG_IDS :
2702 msg->sg.end - shift;
2703out:
2704 msg->data = sg_virt(&msg->sg.data[first_sge]) + start - offset;
2705 msg->data_end = msg->data + bytes;
2706 return 0;
2707}
2708
2709static const struct bpf_func_proto bpf_msg_pull_data_proto = {
2710 .func = bpf_msg_pull_data,
2711 .gpl_only = false,
2712 .ret_type = RET_INTEGER,
2713 .arg1_type = ARG_PTR_TO_CTX,
2714 .arg2_type = ARG_ANYTHING,
2715 .arg3_type = ARG_ANYTHING,
2716 .arg4_type = ARG_ANYTHING,
2717};
2718
2719BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start,
2720 u32, len, u64, flags)
2721{
2722 struct scatterlist sge, nsge, nnsge, rsge = {0}, *psge;
2723 u32 new, i = 0, l = 0, space, copy = 0, offset = 0;
2724 u8 *raw, *to, *from;
2725 struct page *page;
2726
2727 if (unlikely(flags))
2728 return -EINVAL;
2729
2730
2731 i = msg->sg.start;
2732 do {
2733 offset += l;
2734 l = sk_msg_elem(msg, i)->length;
2735
2736 if (start < offset + l)
2737 break;
2738 sk_msg_iter_var_next(i);
2739 } while (i != msg->sg.end);
2740
2741 if (start >= offset + l)
2742 return -EINVAL;
2743
2744 space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);
2745
2746
2747
2748
2749
2750
2751
2752
2753 if (!space || (space == 1 && start != offset))
2754 copy = msg->sg.data[i].length;
2755
2756 page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP,
2757 get_order(copy + len));
2758 if (unlikely(!page))
2759 return -ENOMEM;
2760
2761 if (copy) {
2762 int front, back;
2763
2764 raw = page_address(page);
2765
2766 psge = sk_msg_elem(msg, i);
2767 front = start - offset;
2768 back = psge->length - front;
2769 from = sg_virt(psge);
2770
2771 if (front)
2772 memcpy(raw, from, front);
2773
2774 if (back) {
2775 from += front;
2776 to = raw + front + len;
2777
2778 memcpy(to, from, back);
2779 }
2780
2781 put_page(sg_page(psge));
2782 } else if (start - offset) {
2783 psge = sk_msg_elem(msg, i);
2784 rsge = sk_msg_elem_cpy(msg, i);
2785
2786 psge->length = start - offset;
2787 rsge.length -= psge->length;
2788 rsge.offset += start;
2789
2790 sk_msg_iter_var_next(i);
2791 sg_unmark_end(psge);
2792 sg_unmark_end(&rsge);
2793 sk_msg_iter_next(msg, end);
2794 }
2795
2796
2797 new = i;
2798
2799
2800 if (!copy) {
2801 sge = sk_msg_elem_cpy(msg, i);
2802
2803 sk_msg_iter_var_next(i);
2804 sg_unmark_end(&sge);
2805 sk_msg_iter_next(msg, end);
2806
2807 nsge = sk_msg_elem_cpy(msg, i);
2808 if (rsge.length) {
2809 sk_msg_iter_var_next(i);
2810 nnsge = sk_msg_elem_cpy(msg, i);
2811 }
2812
2813 while (i != msg->sg.end) {
2814 msg->sg.data[i] = sge;
2815 sge = nsge;
2816 sk_msg_iter_var_next(i);
2817 if (rsge.length) {
2818 nsge = nnsge;
2819 nnsge = sk_msg_elem_cpy(msg, i);
2820 } else {
2821 nsge = sk_msg_elem_cpy(msg, i);
2822 }
2823 }
2824 }
2825
2826
2827 sk_mem_charge(msg->sk, len);
2828 msg->sg.size += len;
2829 __clear_bit(new, &msg->sg.copy);
2830 sg_set_page(&msg->sg.data[new], page, len + copy, 0);
2831 if (rsge.length) {
2832 get_page(sg_page(&rsge));
2833 sk_msg_iter_var_next(new);
2834 msg->sg.data[new] = rsge;
2835 }
2836
2837 sk_msg_compute_data_pointers(msg);
2838 return 0;
2839}
2840
2841static const struct bpf_func_proto bpf_msg_push_data_proto = {
2842 .func = bpf_msg_push_data,
2843 .gpl_only = false,
2844 .ret_type = RET_INTEGER,
2845 .arg1_type = ARG_PTR_TO_CTX,
2846 .arg2_type = ARG_ANYTHING,
2847 .arg3_type = ARG_ANYTHING,
2848 .arg4_type = ARG_ANYTHING,
2849};
2850
2851static void sk_msg_shift_left(struct sk_msg *msg, int i)
2852{
2853 int prev;
2854
2855 do {
2856 prev = i;
2857 sk_msg_iter_var_next(i);
2858 msg->sg.data[prev] = msg->sg.data[i];
2859 } while (i != msg->sg.end);
2860
2861 sk_msg_iter_prev(msg, end);
2862}
2863
2864static void sk_msg_shift_right(struct sk_msg *msg, int i)
2865{
2866 struct scatterlist tmp, sge;
2867
2868 sk_msg_iter_next(msg, end);
2869 sge = sk_msg_elem_cpy(msg, i);
2870 sk_msg_iter_var_next(i);
2871 tmp = sk_msg_elem_cpy(msg, i);
2872
2873 while (i != msg->sg.end) {
2874 msg->sg.data[i] = sge;
2875 sk_msg_iter_var_next(i);
2876 sge = tmp;
2877 tmp = sk_msg_elem_cpy(msg, i);
2878 }
2879}
2880
2881BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start,
2882 u32, len, u64, flags)
2883{
2884 u32 i = 0, l = 0, space, offset = 0;
2885 u64 last = start + len;
2886 int pop;
2887
2888 if (unlikely(flags))
2889 return -EINVAL;
2890
2891
2892 i = msg->sg.start;
2893 do {
2894 offset += l;
2895 l = sk_msg_elem(msg, i)->length;
2896
2897 if (start < offset + l)
2898 break;
2899 sk_msg_iter_var_next(i);
2900 } while (i != msg->sg.end);
2901
2902
2903 if (start >= offset + l || last >= msg->sg.size)
2904 return -EINVAL;
2905
2906 space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);
2907
2908 pop = len;
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930 if (start != offset) {
2931 struct scatterlist *nsge, *sge = sk_msg_elem(msg, i);
2932 int a = start;
2933 int b = sge->length - pop - a;
2934
2935 sk_msg_iter_var_next(i);
2936
2937 if (pop < sge->length - a) {
2938 if (space) {
2939 sge->length = a;
2940 sk_msg_shift_right(msg, i);
2941 nsge = sk_msg_elem(msg, i);
2942 get_page(sg_page(sge));
2943 sg_set_page(nsge,
2944 sg_page(sge),
2945 b, sge->offset + pop + a);
2946 } else {
2947 struct page *page, *orig;
2948 u8 *to, *from;
2949
2950 page = alloc_pages(__GFP_NOWARN |
2951 __GFP_COMP | GFP_ATOMIC,
2952 get_order(a + b));
2953 if (unlikely(!page))
2954 return -ENOMEM;
2955
2956 sge->length = a;
2957 orig = sg_page(sge);
2958 from = sg_virt(sge);
2959 to = page_address(page);
2960 memcpy(to, from, a);
2961 memcpy(to + a, from + a + pop, b);
2962 sg_set_page(sge, page, a + b, 0);
2963 put_page(orig);
2964 }
2965 pop = 0;
2966 } else if (pop >= sge->length - a) {
2967 pop -= (sge->length - a);
2968 sge->length = a;
2969 }
2970 }
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989 while (pop) {
2990 struct scatterlist *sge = sk_msg_elem(msg, i);
2991
2992 if (pop < sge->length) {
2993 sge->length -= pop;
2994 sge->offset += pop;
2995 pop = 0;
2996 } else {
2997 pop -= sge->length;
2998 sk_msg_shift_left(msg, i);
2999 }
3000 sk_msg_iter_var_next(i);
3001 }
3002
3003 sk_mem_uncharge(msg->sk, len - pop);
3004 msg->sg.size -= (len - pop);
3005 sk_msg_compute_data_pointers(msg);
3006 return 0;
3007}
3008
3009static const struct bpf_func_proto bpf_msg_pop_data_proto = {
3010 .func = bpf_msg_pop_data,
3011 .gpl_only = false,
3012 .ret_type = RET_INTEGER,
3013 .arg1_type = ARG_PTR_TO_CTX,
3014 .arg2_type = ARG_ANYTHING,
3015 .arg3_type = ARG_ANYTHING,
3016 .arg4_type = ARG_ANYTHING,
3017};
3018
3019#ifdef CONFIG_CGROUP_NET_CLASSID
3020BPF_CALL_0(bpf_get_cgroup_classid_curr)
3021{
3022 return __task_get_classid(current);
3023}
3024
3025static const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = {
3026 .func = bpf_get_cgroup_classid_curr,
3027 .gpl_only = false,
3028 .ret_type = RET_INTEGER,
3029};
3030
3031BPF_CALL_1(bpf_skb_cgroup_classid, const struct sk_buff *, skb)
3032{
3033 struct sock *sk = skb_to_full_sk(skb);
3034
3035 if (!sk || !sk_fullsock(sk))
3036 return 0;
3037
3038 return sock_cgroup_classid(&sk->sk_cgrp_data);
3039}
3040
3041static const struct bpf_func_proto bpf_skb_cgroup_classid_proto = {
3042 .func = bpf_skb_cgroup_classid,
3043 .gpl_only = false,
3044 .ret_type = RET_INTEGER,
3045 .arg1_type = ARG_PTR_TO_CTX,
3046};
3047#endif
3048
3049BPF_CALL_1(bpf_get_cgroup_classid, const struct sk_buff *, skb)
3050{
3051 return task_get_classid(skb);
3052}
3053
3054static const struct bpf_func_proto bpf_get_cgroup_classid_proto = {
3055 .func = bpf_get_cgroup_classid,
3056 .gpl_only = false,
3057 .ret_type = RET_INTEGER,
3058 .arg1_type = ARG_PTR_TO_CTX,
3059};
3060
3061BPF_CALL_1(bpf_get_route_realm, const struct sk_buff *, skb)
3062{
3063 return dst_tclassid(skb);
3064}
3065
3066static const struct bpf_func_proto bpf_get_route_realm_proto = {
3067 .func = bpf_get_route_realm,
3068 .gpl_only = false,
3069 .ret_type = RET_INTEGER,
3070 .arg1_type = ARG_PTR_TO_CTX,
3071};
3072
3073BPF_CALL_1(bpf_get_hash_recalc, struct sk_buff *, skb)
3074{
3075
3076
3077
3078
3079
3080 return skb_get_hash(skb);
3081}
3082
3083static const struct bpf_func_proto bpf_get_hash_recalc_proto = {
3084 .func = bpf_get_hash_recalc,
3085 .gpl_only = false,
3086 .ret_type = RET_INTEGER,
3087 .arg1_type = ARG_PTR_TO_CTX,
3088};
3089
3090BPF_CALL_1(bpf_set_hash_invalid, struct sk_buff *, skb)
3091{
3092
3093
3094
3095 skb_clear_hash(skb);
3096 return 0;
3097}
3098
3099static const struct bpf_func_proto bpf_set_hash_invalid_proto = {
3100 .func = bpf_set_hash_invalid,
3101 .gpl_only = false,
3102 .ret_type = RET_INTEGER,
3103 .arg1_type = ARG_PTR_TO_CTX,
3104};
3105
3106BPF_CALL_2(bpf_set_hash, struct sk_buff *, skb, u32, hash)
3107{
3108
3109
3110
3111
3112 __skb_set_sw_hash(skb, hash, true);
3113 return 0;
3114}
3115
3116static const struct bpf_func_proto bpf_set_hash_proto = {
3117 .func = bpf_set_hash,
3118 .gpl_only = false,
3119 .ret_type = RET_INTEGER,
3120 .arg1_type = ARG_PTR_TO_CTX,
3121 .arg2_type = ARG_ANYTHING,
3122};
3123
3124BPF_CALL_3(bpf_skb_vlan_push, struct sk_buff *, skb, __be16, vlan_proto,
3125 u16, vlan_tci)
3126{
3127 int ret;
3128
3129 if (unlikely(vlan_proto != htons(ETH_P_8021Q) &&
3130 vlan_proto != htons(ETH_P_8021AD)))
3131 vlan_proto = htons(ETH_P_8021Q);
3132
3133 bpf_push_mac_rcsum(skb);
3134 ret = skb_vlan_push(skb, vlan_proto, vlan_tci);
3135 bpf_pull_mac_rcsum(skb);
3136
3137 bpf_compute_data_pointers(skb);
3138 return ret;
3139}
3140
3141static const struct bpf_func_proto bpf_skb_vlan_push_proto = {
3142 .func = bpf_skb_vlan_push,
3143 .gpl_only = false,
3144 .ret_type = RET_INTEGER,
3145 .arg1_type = ARG_PTR_TO_CTX,
3146 .arg2_type = ARG_ANYTHING,
3147 .arg3_type = ARG_ANYTHING,
3148};
3149
3150BPF_CALL_1(bpf_skb_vlan_pop, struct sk_buff *, skb)
3151{
3152 int ret;
3153
3154 bpf_push_mac_rcsum(skb);
3155 ret = skb_vlan_pop(skb);
3156 bpf_pull_mac_rcsum(skb);
3157
3158 bpf_compute_data_pointers(skb);
3159 return ret;
3160}
3161
3162static const struct bpf_func_proto bpf_skb_vlan_pop_proto = {
3163 .func = bpf_skb_vlan_pop,
3164 .gpl_only = false,
3165 .ret_type = RET_INTEGER,
3166 .arg1_type = ARG_PTR_TO_CTX,
3167};
3168
3169static int bpf_skb_generic_push(struct sk_buff *skb, u32 off, u32 len)
3170{
3171
3172
3173
3174 skb_push(skb, len);
3175 memmove(skb->data, skb->data + len, off);
3176 memset(skb->data + off, 0, len);
3177
3178
3179
3180
3181
3182
3183 return 0;
3184}
3185
3186static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len)
3187{
3188
3189
3190
3191 if (unlikely(!pskb_may_pull(skb, off + len)))
3192 return -ENOMEM;
3193
3194 skb_postpull_rcsum(skb, skb->data + off, len);
3195 memmove(skb->data + len, skb->data, off);
3196 __skb_pull(skb, len);
3197
3198 return 0;
3199}
3200
3201static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len)
3202{
3203 bool trans_same = skb->transport_header == skb->network_header;
3204 int ret;
3205
3206
3207
3208
3209
3210 ret = bpf_skb_generic_push(skb, off, len);
3211 if (likely(!ret)) {
3212 skb->mac_header -= len;
3213 skb->network_header -= len;
3214 if (trans_same)
3215 skb->transport_header = skb->network_header;
3216 }
3217
3218 return ret;
3219}
3220
3221static int bpf_skb_net_hdr_pop(struct sk_buff *skb, u32 off, u32 len)
3222{
3223 bool trans_same = skb->transport_header == skb->network_header;
3224 int ret;
3225
3226
3227 ret = bpf_skb_generic_pop(skb, off, len);
3228 if (likely(!ret)) {
3229 skb->mac_header += len;
3230 skb->network_header += len;
3231 if (trans_same)
3232 skb->transport_header = skb->network_header;
3233 }
3234
3235 return ret;
3236}
3237
3238static int bpf_skb_proto_4_to_6(struct sk_buff *skb)
3239{
3240 const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
3241 u32 off = skb_mac_header_len(skb);
3242 int ret;
3243
3244 ret = skb_cow(skb, len_diff);
3245 if (unlikely(ret < 0))
3246 return ret;
3247
3248 ret = bpf_skb_net_hdr_push(skb, off, len_diff);
3249 if (unlikely(ret < 0))
3250 return ret;
3251
3252 if (skb_is_gso(skb)) {
3253 struct skb_shared_info *shinfo = skb_shinfo(skb);
3254
3255
3256 if (shinfo->gso_type & SKB_GSO_TCPV4) {
3257 shinfo->gso_type &= ~SKB_GSO_TCPV4;
3258 shinfo->gso_type |= SKB_GSO_TCPV6;
3259 }
3260 }
3261
3262 skb->protocol = htons(ETH_P_IPV6);
3263 skb_clear_hash(skb);
3264
3265 return 0;
3266}
3267
3268static int bpf_skb_proto_6_to_4(struct sk_buff *skb)
3269{
3270 const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
3271 u32 off = skb_mac_header_len(skb);
3272 int ret;
3273
3274 ret = skb_unclone(skb, GFP_ATOMIC);
3275 if (unlikely(ret < 0))
3276 return ret;
3277
3278 ret = bpf_skb_net_hdr_pop(skb, off, len_diff);
3279 if (unlikely(ret < 0))
3280 return ret;
3281
3282 if (skb_is_gso(skb)) {
3283 struct skb_shared_info *shinfo = skb_shinfo(skb);
3284
3285
3286 if (shinfo->gso_type & SKB_GSO_TCPV6) {
3287 shinfo->gso_type &= ~SKB_GSO_TCPV6;
3288 shinfo->gso_type |= SKB_GSO_TCPV4;
3289 }
3290 }
3291
3292 skb->protocol = htons(ETH_P_IP);
3293 skb_clear_hash(skb);
3294
3295 return 0;
3296}
3297
3298static int bpf_skb_proto_xlat(struct sk_buff *skb, __be16 to_proto)
3299{
3300 __be16 from_proto = skb->protocol;
3301
3302 if (from_proto == htons(ETH_P_IP) &&
3303 to_proto == htons(ETH_P_IPV6))
3304 return bpf_skb_proto_4_to_6(skb);
3305
3306 if (from_proto == htons(ETH_P_IPV6) &&
3307 to_proto == htons(ETH_P_IP))
3308 return bpf_skb_proto_6_to_4(skb);
3309
3310 return -ENOTSUPP;
3311}
3312
3313BPF_CALL_3(bpf_skb_change_proto, struct sk_buff *, skb, __be16, proto,
3314 u64, flags)
3315{
3316 int ret;
3317
3318 if (unlikely(flags))
3319 return -EINVAL;
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338 ret = bpf_skb_proto_xlat(skb, proto);
3339 bpf_compute_data_pointers(skb);
3340 return ret;
3341}
3342
3343static const struct bpf_func_proto bpf_skb_change_proto_proto = {
3344 .func = bpf_skb_change_proto,
3345 .gpl_only = false,
3346 .ret_type = RET_INTEGER,
3347 .arg1_type = ARG_PTR_TO_CTX,
3348 .arg2_type = ARG_ANYTHING,
3349 .arg3_type = ARG_ANYTHING,
3350};
3351
3352BPF_CALL_2(bpf_skb_change_type, struct sk_buff *, skb, u32, pkt_type)
3353{
3354
3355 if (unlikely(!skb_pkt_type_ok(skb->pkt_type) ||
3356 !skb_pkt_type_ok(pkt_type)))
3357 return -EINVAL;
3358
3359 skb->pkt_type = pkt_type;
3360 return 0;
3361}
3362
3363static const struct bpf_func_proto bpf_skb_change_type_proto = {
3364 .func = bpf_skb_change_type,
3365 .gpl_only = false,
3366 .ret_type = RET_INTEGER,
3367 .arg1_type = ARG_PTR_TO_CTX,
3368 .arg2_type = ARG_ANYTHING,
3369};
3370
3371static u32 bpf_skb_net_base_len(const struct sk_buff *skb)
3372{
3373 switch (skb->protocol) {
3374 case htons(ETH_P_IP):
3375 return sizeof(struct iphdr);
3376 case htons(ETH_P_IPV6):
3377 return sizeof(struct ipv6hdr);
3378 default:
3379 return ~0U;
3380 }
3381}
3382
3383#define BPF_F_ADJ_ROOM_ENCAP_L3_MASK (BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 | \
3384 BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3385
3386#define BPF_F_ADJ_ROOM_MASK (BPF_F_ADJ_ROOM_FIXED_GSO | \
3387 BPF_F_ADJ_ROOM_ENCAP_L3_MASK | \
3388 BPF_F_ADJ_ROOM_ENCAP_L4_GRE | \
3389 BPF_F_ADJ_ROOM_ENCAP_L4_UDP | \
3390 BPF_F_ADJ_ROOM_ENCAP_L2_ETH | \
3391 BPF_F_ADJ_ROOM_ENCAP_L2( \
3392 BPF_ADJ_ROOM_ENCAP_L2_MASK))
3393
3394static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff,
3395 u64 flags)
3396{
3397 u8 inner_mac_len = flags >> BPF_ADJ_ROOM_ENCAP_L2_SHIFT;
3398 bool encap = flags & BPF_F_ADJ_ROOM_ENCAP_L3_MASK;
3399 u16 mac_len = 0, inner_net = 0, inner_trans = 0;
3400 unsigned int gso_type = SKB_GSO_DODGY;
3401 int ret;
3402
3403 if (skb_is_gso(skb) && !skb_is_gso_tcp(skb)) {
3404
3405 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ||
3406 !(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3407 return -ENOTSUPP;
3408 }
3409
3410 ret = skb_cow_head(skb, len_diff);
3411 if (unlikely(ret < 0))
3412 return ret;
3413
3414 if (encap) {
3415 if (skb->protocol != htons(ETH_P_IP) &&
3416 skb->protocol != htons(ETH_P_IPV6))
3417 return -ENOTSUPP;
3418
3419 if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 &&
3420 flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3421 return -EINVAL;
3422
3423 if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE &&
3424 flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP)
3425 return -EINVAL;
3426
3427 if (flags & BPF_F_ADJ_ROOM_ENCAP_L2_ETH &&
3428 inner_mac_len < ETH_HLEN)
3429 return -EINVAL;
3430
3431 if (skb->encapsulation)
3432 return -EALREADY;
3433
3434 mac_len = skb->network_header - skb->mac_header;
3435 inner_net = skb->network_header;
3436 if (inner_mac_len > len_diff)
3437 return -EINVAL;
3438 inner_trans = skb->transport_header;
3439 }
3440
3441 ret = bpf_skb_net_hdr_push(skb, off, len_diff);
3442 if (unlikely(ret < 0))
3443 return ret;
3444
3445 if (encap) {
3446 skb->inner_mac_header = inner_net - inner_mac_len;
3447 skb->inner_network_header = inner_net;
3448 skb->inner_transport_header = inner_trans;
3449
3450 if (flags & BPF_F_ADJ_ROOM_ENCAP_L2_ETH)
3451 skb_set_inner_protocol(skb, htons(ETH_P_TEB));
3452 else
3453 skb_set_inner_protocol(skb, skb->protocol);
3454
3455 skb->encapsulation = 1;
3456 skb_set_network_header(skb, mac_len);
3457
3458 if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP)
3459 gso_type |= SKB_GSO_UDP_TUNNEL;
3460 else if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE)
3461 gso_type |= SKB_GSO_GRE;
3462 else if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3463 gso_type |= SKB_GSO_IPXIP6;
3464 else if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4)
3465 gso_type |= SKB_GSO_IPXIP4;
3466
3467 if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE ||
3468 flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP) {
3469 int nh_len = flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 ?
3470 sizeof(struct ipv6hdr) :
3471 sizeof(struct iphdr);
3472
3473 skb_set_transport_header(skb, mac_len + nh_len);
3474 }
3475
3476
3477 if (skb->protocol == htons(ETH_P_IP) &&
3478 flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3479 skb->protocol = htons(ETH_P_IPV6);
3480 else if (skb->protocol == htons(ETH_P_IPV6) &&
3481 flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4)
3482 skb->protocol = htons(ETH_P_IP);
3483 }
3484
3485 if (skb_is_gso(skb)) {
3486 struct skb_shared_info *shinfo = skb_shinfo(skb);
3487
3488
3489 if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3490 skb_decrease_gso_size(shinfo, len_diff);
3491
3492
3493 shinfo->gso_type |= gso_type;
3494 shinfo->gso_segs = 0;
3495 }
3496
3497 return 0;
3498}
3499
3500static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff,
3501 u64 flags)
3502{
3503 int ret;
3504
3505 if (unlikely(flags & ~(BPF_F_ADJ_ROOM_FIXED_GSO |
3506 BPF_F_ADJ_ROOM_NO_CSUM_RESET)))
3507 return -EINVAL;
3508
3509 if (skb_is_gso(skb) && !skb_is_gso_tcp(skb)) {
3510
3511 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ||
3512 !(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3513 return -ENOTSUPP;
3514 }
3515
3516 ret = skb_unclone(skb, GFP_ATOMIC);
3517 if (unlikely(ret < 0))
3518 return ret;
3519
3520 ret = bpf_skb_net_hdr_pop(skb, off, len_diff);
3521 if (unlikely(ret < 0))
3522 return ret;
3523
3524 if (skb_is_gso(skb)) {
3525 struct skb_shared_info *shinfo = skb_shinfo(skb);
3526
3527
3528 if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3529 skb_increase_gso_size(shinfo, len_diff);
3530
3531
3532 shinfo->gso_type |= SKB_GSO_DODGY;
3533 shinfo->gso_segs = 0;
3534 }
3535
3536 return 0;
3537}
3538
3539#define BPF_SKB_MAX_LEN SKB_MAX_ALLOC
3540
3541BPF_CALL_4(sk_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
3542 u32, mode, u64, flags)
3543{
3544 u32 len_diff_abs = abs(len_diff);
3545 bool shrink = len_diff < 0;
3546 int ret = 0;
3547
3548 if (unlikely(flags || mode))
3549 return -EINVAL;
3550 if (unlikely(len_diff_abs > 0xfffU))
3551 return -EFAULT;
3552
3553 if (!shrink) {
3554 ret = skb_cow(skb, len_diff);
3555 if (unlikely(ret < 0))
3556 return ret;
3557 __skb_push(skb, len_diff_abs);
3558 memset(skb->data, 0, len_diff_abs);
3559 } else {
3560 if (unlikely(!pskb_may_pull(skb, len_diff_abs)))
3561 return -ENOMEM;
3562 __skb_pull(skb, len_diff_abs);
3563 }
3564 if (tls_sw_has_ctx_rx(skb->sk)) {
3565 struct strp_msg *rxm = strp_msg(skb);
3566
3567 rxm->full_len += len_diff;
3568 }
3569 return ret;
3570}
3571
3572static const struct bpf_func_proto sk_skb_adjust_room_proto = {
3573 .func = sk_skb_adjust_room,
3574 .gpl_only = false,
3575 .ret_type = RET_INTEGER,
3576 .arg1_type = ARG_PTR_TO_CTX,
3577 .arg2_type = ARG_ANYTHING,
3578 .arg3_type = ARG_ANYTHING,
3579 .arg4_type = ARG_ANYTHING,
3580};
3581
3582BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
3583 u32, mode, u64, flags)
3584{
3585 u32 len_cur, len_diff_abs = abs(len_diff);
3586 u32 len_min = bpf_skb_net_base_len(skb);
3587 u32 len_max = BPF_SKB_MAX_LEN;
3588 __be16 proto = skb->protocol;
3589 bool shrink = len_diff < 0;
3590 u32 off;
3591 int ret;
3592
3593 if (unlikely(flags & ~(BPF_F_ADJ_ROOM_MASK |
3594 BPF_F_ADJ_ROOM_NO_CSUM_RESET)))
3595 return -EINVAL;
3596 if (unlikely(len_diff_abs > 0xfffU))
3597 return -EFAULT;
3598 if (unlikely(proto != htons(ETH_P_IP) &&
3599 proto != htons(ETH_P_IPV6)))
3600 return -ENOTSUPP;
3601
3602 off = skb_mac_header_len(skb);
3603 switch (mode) {
3604 case BPF_ADJ_ROOM_NET:
3605 off += bpf_skb_net_base_len(skb);
3606 break;
3607 case BPF_ADJ_ROOM_MAC:
3608 break;
3609 default:
3610 return -ENOTSUPP;
3611 }
3612
3613 len_cur = skb->len - skb_network_offset(skb);
3614 if ((shrink && (len_diff_abs >= len_cur ||
3615 len_cur - len_diff_abs < len_min)) ||
3616 (!shrink && (skb->len + len_diff_abs > len_max &&
3617 !skb_is_gso(skb))))
3618 return -ENOTSUPP;
3619
3620 ret = shrink ? bpf_skb_net_shrink(skb, off, len_diff_abs, flags) :
3621 bpf_skb_net_grow(skb, off, len_diff_abs, flags);
3622 if (!ret && !(flags & BPF_F_ADJ_ROOM_NO_CSUM_RESET))
3623 __skb_reset_checksum_unnecessary(skb);
3624
3625 bpf_compute_data_pointers(skb);
3626 return ret;
3627}
3628
3629static const struct bpf_func_proto bpf_skb_adjust_room_proto = {
3630 .func = bpf_skb_adjust_room,
3631 .gpl_only = false,
3632 .ret_type = RET_INTEGER,
3633 .arg1_type = ARG_PTR_TO_CTX,
3634 .arg2_type = ARG_ANYTHING,
3635 .arg3_type = ARG_ANYTHING,
3636 .arg4_type = ARG_ANYTHING,
3637};
3638
3639static u32 __bpf_skb_min_len(const struct sk_buff *skb)
3640{
3641 u32 min_len = skb_network_offset(skb);
3642
3643 if (skb_transport_header_was_set(skb))
3644 min_len = skb_transport_offset(skb);
3645 if (skb->ip_summed == CHECKSUM_PARTIAL)
3646 min_len = skb_checksum_start_offset(skb) +
3647 skb->csum_offset + sizeof(__sum16);
3648 return min_len;
3649}
3650
3651static int bpf_skb_grow_rcsum(struct sk_buff *skb, unsigned int new_len)
3652{
3653 unsigned int old_len = skb->len;
3654 int ret;
3655
3656 ret = __skb_grow_rcsum(skb, new_len);
3657 if (!ret)
3658 memset(skb->data + old_len, 0, new_len - old_len);
3659 return ret;
3660}
3661
3662static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len)
3663{
3664 return __skb_trim_rcsum(skb, new_len);
3665}
3666
3667static inline int __bpf_skb_change_tail(struct sk_buff *skb, u32 new_len,
3668 u64 flags)
3669{
3670 u32 max_len = BPF_SKB_MAX_LEN;
3671 u32 min_len = __bpf_skb_min_len(skb);
3672 int ret;
3673
3674 if (unlikely(flags || new_len > max_len || new_len < min_len))
3675 return -EINVAL;
3676 if (skb->encapsulation)
3677 return -ENOTSUPP;
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695 ret = __bpf_try_make_writable(skb, skb->len);
3696 if (!ret) {
3697 if (new_len > skb->len)
3698 ret = bpf_skb_grow_rcsum(skb, new_len);
3699 else if (new_len < skb->len)
3700 ret = bpf_skb_trim_rcsum(skb, new_len);
3701 if (!ret && skb_is_gso(skb))
3702 skb_gso_reset(skb);
3703 }
3704 return ret;
3705}
3706
3707BPF_CALL_3(bpf_skb_change_tail, struct sk_buff *, skb, u32, new_len,
3708 u64, flags)
3709{
3710 int ret = __bpf_skb_change_tail(skb, new_len, flags);
3711
3712 bpf_compute_data_pointers(skb);
3713 return ret;
3714}
3715
3716static const struct bpf_func_proto bpf_skb_change_tail_proto = {
3717 .func = bpf_skb_change_tail,
3718 .gpl_only = false,
3719 .ret_type = RET_INTEGER,
3720 .arg1_type = ARG_PTR_TO_CTX,
3721 .arg2_type = ARG_ANYTHING,
3722 .arg3_type = ARG_ANYTHING,
3723};
3724
3725BPF_CALL_3(sk_skb_change_tail, struct sk_buff *, skb, u32, new_len,
3726 u64, flags)
3727{
3728 return __bpf_skb_change_tail(skb, new_len, flags);
3729}
3730
3731static const struct bpf_func_proto sk_skb_change_tail_proto = {
3732 .func = sk_skb_change_tail,
3733 .gpl_only = false,
3734 .ret_type = RET_INTEGER,
3735 .arg1_type = ARG_PTR_TO_CTX,
3736 .arg2_type = ARG_ANYTHING,
3737 .arg3_type = ARG_ANYTHING,
3738};
3739
3740static inline int __bpf_skb_change_head(struct sk_buff *skb, u32 head_room,
3741 u64 flags)
3742{
3743 u32 max_len = BPF_SKB_MAX_LEN;
3744 u32 new_len = skb->len + head_room;
3745 int ret;
3746
3747 if (unlikely(flags || (!skb_is_gso(skb) && new_len > max_len) ||
3748 new_len < skb->len))
3749 return -EINVAL;
3750
3751 ret = skb_cow(skb, head_room);
3752 if (likely(!ret)) {
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762 __skb_push(skb, head_room);
3763 memset(skb->data, 0, head_room);
3764 skb_reset_mac_header(skb);
3765 skb_reset_mac_len(skb);
3766 }
3767
3768 return ret;
3769}
3770
3771BPF_CALL_3(bpf_skb_change_head, struct sk_buff *, skb, u32, head_room,
3772 u64, flags)
3773{
3774 int ret = __bpf_skb_change_head(skb, head_room, flags);
3775
3776 bpf_compute_data_pointers(skb);
3777 return ret;
3778}
3779
3780static const struct bpf_func_proto bpf_skb_change_head_proto = {
3781 .func = bpf_skb_change_head,
3782 .gpl_only = false,
3783 .ret_type = RET_INTEGER,
3784 .arg1_type = ARG_PTR_TO_CTX,
3785 .arg2_type = ARG_ANYTHING,
3786 .arg3_type = ARG_ANYTHING,
3787};
3788
3789BPF_CALL_3(sk_skb_change_head, struct sk_buff *, skb, u32, head_room,
3790 u64, flags)
3791{
3792 return __bpf_skb_change_head(skb, head_room, flags);
3793}
3794
3795static const struct bpf_func_proto sk_skb_change_head_proto = {
3796 .func = sk_skb_change_head,
3797 .gpl_only = false,
3798 .ret_type = RET_INTEGER,
3799 .arg1_type = ARG_PTR_TO_CTX,
3800 .arg2_type = ARG_ANYTHING,
3801 .arg3_type = ARG_ANYTHING,
3802};
3803static unsigned long xdp_get_metalen(const struct xdp_buff *xdp)
3804{
3805 return xdp_data_meta_unsupported(xdp) ? 0 :
3806 xdp->data - xdp->data_meta;
3807}
3808
3809BPF_CALL_2(bpf_xdp_adjust_head, struct xdp_buff *, xdp, int, offset)
3810{
3811 void *xdp_frame_end = xdp->data_hard_start + sizeof(struct xdp_frame);
3812 unsigned long metalen = xdp_get_metalen(xdp);
3813 void *data_start = xdp_frame_end + metalen;
3814 void *data = xdp->data + offset;
3815
3816 if (unlikely(data < data_start ||
3817 data > xdp->data_end - ETH_HLEN))
3818 return -EINVAL;
3819
3820 if (metalen)
3821 memmove(xdp->data_meta + offset,
3822 xdp->data_meta, metalen);
3823 xdp->data_meta += offset;
3824 xdp->data = data;
3825
3826 return 0;
3827}
3828
3829static const struct bpf_func_proto bpf_xdp_adjust_head_proto = {
3830 .func = bpf_xdp_adjust_head,
3831 .gpl_only = false,
3832 .ret_type = RET_INTEGER,
3833 .arg1_type = ARG_PTR_TO_CTX,
3834 .arg2_type = ARG_ANYTHING,
3835};
3836
3837BPF_CALL_2(bpf_xdp_adjust_tail, struct xdp_buff *, xdp, int, offset)
3838{
3839 void *data_hard_end = xdp_data_hard_end(xdp);
3840 void *data_end = xdp->data_end + offset;
3841
3842
3843 if (unlikely(data_end > data_hard_end))
3844 return -EINVAL;
3845
3846
3847 if (unlikely(xdp->frame_sz > PAGE_SIZE)) {
3848 WARN_ONCE(1, "Too BIG xdp->frame_sz = %d\n", xdp->frame_sz);
3849 return -EINVAL;
3850 }
3851
3852 if (unlikely(data_end < xdp->data + ETH_HLEN))
3853 return -EINVAL;
3854
3855
3856 if (offset > 0)
3857 memset(xdp->data_end, 0, offset);
3858
3859 xdp->data_end = data_end;
3860
3861 return 0;
3862}
3863
3864static const struct bpf_func_proto bpf_xdp_adjust_tail_proto = {
3865 .func = bpf_xdp_adjust_tail,
3866 .gpl_only = false,
3867 .ret_type = RET_INTEGER,
3868 .arg1_type = ARG_PTR_TO_CTX,
3869 .arg2_type = ARG_ANYTHING,
3870};
3871
3872BPF_CALL_2(bpf_xdp_adjust_meta, struct xdp_buff *, xdp, int, offset)
3873{
3874 void *xdp_frame_end = xdp->data_hard_start + sizeof(struct xdp_frame);
3875 void *meta = xdp->data_meta + offset;
3876 unsigned long metalen = xdp->data - meta;
3877
3878 if (xdp_data_meta_unsupported(xdp))
3879 return -ENOTSUPP;
3880 if (unlikely(meta < xdp_frame_end ||
3881 meta > xdp->data))
3882 return -EINVAL;
3883 if (unlikely((metalen & (sizeof(__u32) - 1)) ||
3884 (metalen > 32)))
3885 return -EACCES;
3886
3887 xdp->data_meta = meta;
3888
3889 return 0;
3890}
3891
3892static const struct bpf_func_proto bpf_xdp_adjust_meta_proto = {
3893 .func = bpf_xdp_adjust_meta,
3894 .gpl_only = false,
3895 .ret_type = RET_INTEGER,
3896 .arg1_type = ARG_PTR_TO_CTX,
3897 .arg2_type = ARG_ANYTHING,
3898};
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928void xdp_do_flush(void)
3929{
3930 __dev_flush();
3931 __cpu_map_flush();
3932 __xsk_map_flush();
3933}
3934EXPORT_SYMBOL_GPL(xdp_do_flush);
3935
3936void bpf_clear_redirect_map(struct bpf_map *map)
3937{
3938 struct bpf_redirect_info *ri;
3939 int cpu;
3940
3941 for_each_possible_cpu(cpu) {
3942 ri = per_cpu_ptr(&bpf_redirect_info, cpu);
3943
3944
3945
3946
3947
3948 if (unlikely(READ_ONCE(ri->map) == map))
3949 cmpxchg(&ri->map, map, NULL);
3950 }
3951}
3952
3953int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp,
3954 struct bpf_prog *xdp_prog)
3955{
3956 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
3957 enum bpf_map_type map_type = ri->map_type;
3958 void *fwd = ri->tgt_value;
3959 u32 map_id = ri->map_id;
3960 struct bpf_map *map;
3961 int err;
3962
3963 ri->map_id = 0;
3964 ri->map_type = BPF_MAP_TYPE_UNSPEC;
3965
3966 switch (map_type) {
3967 case BPF_MAP_TYPE_DEVMAP:
3968 fallthrough;
3969 case BPF_MAP_TYPE_DEVMAP_HASH:
3970 map = READ_ONCE(ri->map);
3971 if (unlikely(map)) {
3972 WRITE_ONCE(ri->map, NULL);
3973 err = dev_map_enqueue_multi(xdp, dev, map,
3974 ri->flags & BPF_F_EXCLUDE_INGRESS);
3975 } else {
3976 err = dev_map_enqueue(fwd, xdp, dev);
3977 }
3978 break;
3979 case BPF_MAP_TYPE_CPUMAP:
3980 err = cpu_map_enqueue(fwd, xdp, dev);
3981 break;
3982 case BPF_MAP_TYPE_XSKMAP:
3983 err = __xsk_map_redirect(fwd, xdp);
3984 break;
3985 case BPF_MAP_TYPE_UNSPEC:
3986 if (map_id == INT_MAX) {
3987 fwd = dev_get_by_index_rcu(dev_net(dev), ri->tgt_index);
3988 if (unlikely(!fwd)) {
3989 err = -EINVAL;
3990 break;
3991 }
3992 err = dev_xdp_enqueue(fwd, xdp, dev);
3993 break;
3994 }
3995 fallthrough;
3996 default:
3997 err = -EBADRQC;
3998 }
3999
4000 if (unlikely(err))
4001 goto err;
4002
4003 _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
4004 return 0;
4005err:
4006 _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
4007 return err;
4008}
4009EXPORT_SYMBOL_GPL(xdp_do_redirect);
4010
4011static int xdp_do_generic_redirect_map(struct net_device *dev,
4012 struct sk_buff *skb,
4013 struct xdp_buff *xdp,
4014 struct bpf_prog *xdp_prog,
4015 void *fwd,
4016 enum bpf_map_type map_type, u32 map_id)
4017{
4018 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
4019 struct bpf_map *map;
4020 int err;
4021
4022 switch (map_type) {
4023 case BPF_MAP_TYPE_DEVMAP:
4024 fallthrough;
4025 case BPF_MAP_TYPE_DEVMAP_HASH:
4026 map = READ_ONCE(ri->map);
4027 if (unlikely(map)) {
4028 WRITE_ONCE(ri->map, NULL);
4029 err = dev_map_redirect_multi(dev, skb, xdp_prog, map,
4030 ri->flags & BPF_F_EXCLUDE_INGRESS);
4031 } else {
4032 err = dev_map_generic_redirect(fwd, skb, xdp_prog);
4033 }
4034 if (unlikely(err))
4035 goto err;
4036 break;
4037 case BPF_MAP_TYPE_XSKMAP:
4038 err = xsk_generic_rcv(fwd, xdp);
4039 if (err)
4040 goto err;
4041 consume_skb(skb);
4042 break;
4043 default:
4044
4045 err = -EBADRQC;
4046 goto err;
4047 }
4048
4049 _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
4050 return 0;
4051err:
4052 _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
4053 return err;
4054}
4055
4056int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
4057 struct xdp_buff *xdp, struct bpf_prog *xdp_prog)
4058{
4059 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
4060 enum bpf_map_type map_type = ri->map_type;
4061 void *fwd = ri->tgt_value;
4062 u32 map_id = ri->map_id;
4063 int err;
4064
4065 ri->map_id = 0;
4066 ri->map_type = BPF_MAP_TYPE_UNSPEC;
4067
4068 if (map_type == BPF_MAP_TYPE_UNSPEC && map_id == INT_MAX) {
4069 fwd = dev_get_by_index_rcu(dev_net(dev), ri->tgt_index);
4070 if (unlikely(!fwd)) {
4071 err = -EINVAL;
4072 goto err;
4073 }
4074
4075 err = xdp_ok_fwd_dev(fwd, skb->len);
4076 if (unlikely(err))
4077 goto err;
4078
4079 skb->dev = fwd;
4080 _trace_xdp_redirect(dev, xdp_prog, ri->tgt_index);
4081 generic_xdp_tx(skb, xdp_prog);
4082 return 0;
4083 }
4084
4085 return xdp_do_generic_redirect_map(dev, skb, xdp, xdp_prog, fwd, map_type, map_id);
4086err:
4087 _trace_xdp_redirect_err(dev, xdp_prog, ri->tgt_index, err);
4088 return err;
4089}
4090
4091BPF_CALL_2(bpf_xdp_redirect, u32, ifindex, u64, flags)
4092{
4093 struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
4094
4095 if (unlikely(flags))
4096 return XDP_ABORTED;
4097
4098
4099
4100
4101 ri->tgt_index = ifindex;
4102 ri->map_id = INT_MAX;
4103 ri->map_type = BPF_MAP_TYPE_UNSPEC;
4104
4105 return XDP_REDIRECT;
4106}
4107
4108static const struct bpf_func_proto bpf_xdp_redirect_proto = {
4109 .func = bpf_xdp_redirect,
4110 .gpl_only = false,
4111 .ret_type = RET_INTEGER,
4112 .arg1_type = ARG_ANYTHING,
4113 .arg2_type = ARG_ANYTHING,
4114};
4115
4116BPF_CALL_3(bpf_xdp_redirect_map, struct bpf_map *, map, u32, ifindex,
4117 u64, flags)
4118{
4119 return map->ops->map_redirect(map, ifindex, flags);
4120}
4121
4122static const struct bpf_func_proto bpf_xdp_redirect_map_proto = {
4123 .func = bpf_xdp_redirect_map,
4124 .gpl_only = false,
4125 .ret_type = RET_INTEGER,
4126 .arg1_type = ARG_CONST_MAP_PTR,
4127 .arg2_type = ARG_ANYTHING,
4128 .arg3_type = ARG_ANYTHING,
4129};
4130
4131static unsigned long bpf_skb_copy(void *dst_buff, const void *skb,
4132 unsigned long off, unsigned long len)
4133{
4134 void *ptr = skb_header_pointer(skb, off, len, dst_buff);
4135
4136 if (unlikely(!ptr))
4137 return len;
4138 if (ptr != dst_buff)
4139 memcpy(dst_buff, ptr, len);
4140
4141 return 0;
4142}
4143
4144BPF_CALL_5(bpf_skb_event_output, struct sk_buff *, skb, struct bpf_map *, map,
4145 u64, flags, void *, meta, u64, meta_size)
4146{
4147 u64 skb_size = (flags & BPF_F_CTXLEN_MASK) >> 32;
4148
4149 if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
4150 return -EINVAL;
4151 if (unlikely(!skb || skb_size > skb->len))
4152 return -EFAULT;
4153
4154 return bpf_event_output(map, flags, meta, meta_size, skb, skb_size,
4155 bpf_skb_copy);
4156}
4157
4158static const struct bpf_func_proto bpf_skb_event_output_proto = {
4159 .func = bpf_skb_event_output,
4160 .gpl_only = true,
4161 .ret_type = RET_INTEGER,
4162 .arg1_type = ARG_PTR_TO_CTX,
4163 .arg2_type = ARG_CONST_MAP_PTR,
4164 .arg3_type = ARG_ANYTHING,
4165 .arg4_type = ARG_PTR_TO_MEM,
4166 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
4167};
4168
4169BTF_ID_LIST_SINGLE(bpf_skb_output_btf_ids, struct, sk_buff)
4170
4171const struct bpf_func_proto bpf_skb_output_proto = {
4172 .func = bpf_skb_event_output,
4173 .gpl_only = true,
4174 .ret_type = RET_INTEGER,
4175 .arg1_type = ARG_PTR_TO_BTF_ID,
4176 .arg1_btf_id = &bpf_skb_output_btf_ids[0],
4177 .arg2_type = ARG_CONST_MAP_PTR,
4178 .arg3_type = ARG_ANYTHING,
4179 .arg4_type = ARG_PTR_TO_MEM,
4180 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
4181};
4182
4183static unsigned short bpf_tunnel_key_af(u64 flags)
4184{
4185 return flags & BPF_F_TUNINFO_IPV6 ? AF_INET6 : AF_INET;
4186}
4187
4188BPF_CALL_4(bpf_skb_get_tunnel_key, struct sk_buff *, skb, struct bpf_tunnel_key *, to,
4189 u32, size, u64, flags)
4190{
4191 const struct ip_tunnel_info *info = skb_tunnel_info(skb);
4192 u8 compat[sizeof(struct bpf_tunnel_key)];
4193 void *to_orig = to;
4194 int err;
4195
4196 if (unlikely(!info || (flags & ~(BPF_F_TUNINFO_IPV6)))) {
4197 err = -EINVAL;
4198 goto err_clear;
4199 }
4200 if (ip_tunnel_info_af(info) != bpf_tunnel_key_af(flags)) {
4201 err = -EPROTO;
4202 goto err_clear;
4203 }
4204 if (unlikely(size != sizeof(struct bpf_tunnel_key))) {
4205 err = -EINVAL;
4206 switch (size) {
4207 case offsetof(struct bpf_tunnel_key, tunnel_label):
4208 case offsetof(struct bpf_tunnel_key, tunnel_ext):
4209 goto set_compat;
4210 case offsetof(struct bpf_tunnel_key, remote_ipv6[1]):
4211
4212
4213
4214 if (ip_tunnel_info_af(info) != AF_INET)
4215 goto err_clear;
4216set_compat:
4217 to = (struct bpf_tunnel_key *)compat;
4218 break;
4219 default:
4220 goto err_clear;
4221 }
4222 }
4223
4224 to->tunnel_id = be64_to_cpu(info->key.tun_id);
4225 to->tunnel_tos = info->key.tos;
4226 to->tunnel_ttl = info->key.ttl;
4227 to->tunnel_ext = 0;
4228
4229 if (flags & BPF_F_TUNINFO_IPV6) {
4230 memcpy(to->remote_ipv6, &info->key.u.ipv6.src,
4231 sizeof(to->remote_ipv6));
4232 to->tunnel_label = be32_to_cpu(info->key.label);
4233 } else {
4234 to->remote_ipv4 = be32_to_cpu(info->key.u.ipv4.src);
4235 memset(&to->remote_ipv6[1], 0, sizeof(__u32) * 3);
4236 to->tunnel_label = 0;
4237 }
4238
4239 if (unlikely(size != sizeof(struct bpf_tunnel_key)))
4240 memcpy(to_orig, to, size);
4241
4242 return 0;
4243err_clear:
4244 memset(to_orig, 0, size);
4245 return err;
4246}
4247
4248static const struct bpf_func_proto bpf_skb_get_tunnel_key_proto = {
4249 .func = bpf_skb_get_tunnel_key,
4250 .gpl_only = false,
4251 .ret_type = RET_INTEGER,
4252 .arg1_type = ARG_PTR_TO_CTX,
4253 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
4254 .arg3_type = ARG_CONST_SIZE,
4255 .arg4_type = ARG_ANYTHING,
4256};
4257
4258BPF_CALL_3(bpf_skb_get_tunnel_opt, struct sk_buff *, skb, u8 *, to, u32, size)
4259{
4260 const struct ip_tunnel_info *info = skb_tunnel_info(skb);
4261 int err;
4262
4263 if (unlikely(!info ||
4264 !(info->key.tun_flags & TUNNEL_OPTIONS_PRESENT))) {
4265 err = -ENOENT;
4266 goto err_clear;
4267 }
4268 if (unlikely(size < info->options_len)) {
4269 err = -ENOMEM;
4270 goto err_clear;
4271 }
4272
4273 ip_tunnel_info_opts_get(to, info);
4274 if (size > info->options_len)
4275 memset(to + info->options_len, 0, size - info->options_len);
4276
4277 return info->options_len;
4278err_clear:
4279 memset(to, 0, size);
4280 return err;
4281}
4282
4283static const struct bpf_func_proto bpf_skb_get_tunnel_opt_proto = {
4284 .func = bpf_skb_get_tunnel_opt,
4285 .gpl_only = false,
4286 .ret_type = RET_INTEGER,
4287 .arg1_type = ARG_PTR_TO_CTX,
4288 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
4289 .arg3_type = ARG_CONST_SIZE,
4290};
4291
4292static struct metadata_dst __percpu *md_dst;
4293
4294BPF_CALL_4(bpf_skb_set_tunnel_key, struct sk_buff *, skb,
4295 const struct bpf_tunnel_key *, from, u32, size, u64, flags)
4296{
4297 struct metadata_dst *md = this_cpu_ptr(md_dst);
4298 u8 compat[sizeof(struct bpf_tunnel_key)];
4299 struct ip_tunnel_info *info;
4300
4301 if (unlikely(flags & ~(BPF_F_TUNINFO_IPV6 | BPF_F_ZERO_CSUM_TX |
4302 BPF_F_DONT_FRAGMENT | BPF_F_SEQ_NUMBER)))
4303 return -EINVAL;
4304 if (unlikely(size != sizeof(struct bpf_tunnel_key))) {
4305 switch (size) {
4306 case offsetof(struct bpf_tunnel_key, tunnel_label):
4307 case offsetof(struct bpf_tunnel_key, tunnel_ext):
4308 case offsetof(struct bpf_tunnel_key, remote_ipv6[1]):
4309
4310
4311
4312 memcpy(compat, from, size);
4313 memset(compat + size, 0, sizeof(compat) - size);
4314 from = (const struct bpf_tunnel_key *) compat;
4315 break;
4316 default:
4317 return -EINVAL;
4318 }
4319 }
4320 if (unlikely((!(flags & BPF_F_TUNINFO_IPV6) && from->tunnel_label) ||
4321 from->tunnel_ext))
4322 return -EINVAL;
4323
4324 skb_dst_drop(skb);
4325 dst_hold((struct dst_entry *) md);
4326 skb_dst_set(skb, (struct dst_entry *) md);
4327
4328 info = &md->u.tun_info;
4329 memset(info, 0, sizeof(*info));
4330 info->mode = IP_TUNNEL_INFO_TX;
4331
4332 info->key.tun_flags = TUNNEL_KEY | TUNNEL_CSUM | TUNNEL_NOCACHE;
4333 if (flags & BPF_F_DONT_FRAGMENT)
4334 info->key.tun_flags |= TUNNEL_DONT_FRAGMENT;
4335 if (flags & BPF_F_ZERO_CSUM_TX)
4336 info->key.tun_flags &= ~TUNNEL_CSUM;
4337 if (flags & BPF_F_SEQ_NUMBER)
4338 info->key.tun_flags |= TUNNEL_SEQ;
4339
4340 info->key.tun_id = cpu_to_be64(from->tunnel_id);
4341 info->key.tos = from->tunnel_tos;
4342 info->key.ttl = from->tunnel_ttl;
4343
4344 if (flags & BPF_F_TUNINFO_IPV6) {
4345 info->mode |= IP_TUNNEL_INFO_IPV6;
4346 memcpy(&info->key.u.ipv6.dst, from->remote_ipv6,
4347 sizeof(from->remote_ipv6));
4348 info->key.label = cpu_to_be32(from->tunnel_label) &
4349 IPV6_FLOWLABEL_MASK;
4350 } else {
4351 info->key.u.ipv4.dst = cpu_to_be32(from->remote_ipv4);
4352 }
4353
4354 return 0;
4355}
4356
4357static const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = {
4358 .func = bpf_skb_set_tunnel_key,
4359 .gpl_only = false,
4360 .ret_type = RET_INTEGER,
4361 .arg1_type = ARG_PTR_TO_CTX,
4362 .arg2_type = ARG_PTR_TO_MEM,
4363 .arg3_type = ARG_CONST_SIZE,
4364 .arg4_type = ARG_ANYTHING,
4365};
4366
4367BPF_CALL_3(bpf_skb_set_tunnel_opt, struct sk_buff *, skb,
4368 const u8 *, from, u32, size)
4369{
4370 struct ip_tunnel_info *info = skb_tunnel_info(skb);
4371 const struct metadata_dst *md = this_cpu_ptr(md_dst);
4372
4373 if (unlikely(info != &md->u.tun_info || (size & (sizeof(u32) - 1))))
4374 return -EINVAL;
4375 if (unlikely(size > IP_TUNNEL_OPTS_MAX))
4376 return -ENOMEM;
4377
4378 ip_tunnel_info_opts_set(info, from, size, TUNNEL_OPTIONS_PRESENT);
4379
4380 return 0;
4381}
4382
4383static const struct bpf_func_proto bpf_skb_set_tunnel_opt_proto = {
4384 .func = bpf_skb_set_tunnel_opt,
4385 .gpl_only = false,
4386 .ret_type = RET_INTEGER,
4387 .arg1_type = ARG_PTR_TO_CTX,
4388 .arg2_type = ARG_PTR_TO_MEM,
4389 .arg3_type = ARG_CONST_SIZE,
4390};
4391
4392static const struct bpf_func_proto *
4393bpf_get_skb_set_tunnel_proto(enum bpf_func_id which)
4394{
4395 if (!md_dst) {
4396 struct metadata_dst __percpu *tmp;
4397
4398 tmp = metadata_dst_alloc_percpu(IP_TUNNEL_OPTS_MAX,
4399 METADATA_IP_TUNNEL,
4400 GFP_KERNEL);
4401 if (!tmp)
4402 return NULL;
4403 if (cmpxchg(&md_dst, NULL, tmp))
4404 metadata_dst_free_percpu(tmp);
4405 }
4406
4407 switch (which) {
4408 case BPF_FUNC_skb_set_tunnel_key:
4409 return &bpf_skb_set_tunnel_key_proto;
4410 case BPF_FUNC_skb_set_tunnel_opt:
4411 return &bpf_skb_set_tunnel_opt_proto;
4412 default:
4413 return NULL;
4414 }
4415}
4416
4417BPF_CALL_3(bpf_skb_under_cgroup, struct sk_buff *, skb, struct bpf_map *, map,
4418 u32, idx)
4419{
4420 struct bpf_array *array = container_of(map, struct bpf_array, map);
4421 struct cgroup *cgrp;
4422 struct sock *sk;
4423
4424 sk = skb_to_full_sk(skb);
4425 if (!sk || !sk_fullsock(sk))
4426 return -ENOENT;
4427 if (unlikely(idx >= array->map.max_entries))
4428 return -E2BIG;
4429
4430 cgrp = READ_ONCE(array->ptrs[idx]);
4431 if (unlikely(!cgrp))
4432 return -EAGAIN;
4433
4434 return sk_under_cgroup_hierarchy(sk, cgrp);
4435}
4436
4437static const struct bpf_func_proto bpf_skb_under_cgroup_proto = {
4438 .func = bpf_skb_under_cgroup,
4439 .gpl_only = false,
4440 .ret_type = RET_INTEGER,
4441 .arg1_type = ARG_PTR_TO_CTX,
4442 .arg2_type = ARG_CONST_MAP_PTR,
4443 .arg3_type = ARG_ANYTHING,
4444};
4445
4446#ifdef CONFIG_SOCK_CGROUP_DATA
4447static inline u64 __bpf_sk_cgroup_id(struct sock *sk)
4448{
4449 struct cgroup *cgrp;
4450
4451 sk = sk_to_full_sk(sk);
4452 if (!sk || !sk_fullsock(sk))
4453 return 0;
4454
4455 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
4456 return cgroup_id(cgrp);
4457}
4458
4459BPF_CALL_1(bpf_skb_cgroup_id, const struct sk_buff *, skb)
4460{
4461 return __bpf_sk_cgroup_id(skb->sk);
4462}
4463
4464static const struct bpf_func_proto bpf_skb_cgroup_id_proto = {
4465 .func = bpf_skb_cgroup_id,
4466 .gpl_only = false,
4467 .ret_type = RET_INTEGER,
4468 .arg1_type = ARG_PTR_TO_CTX,
4469};
4470
4471static inline u64 __bpf_sk_ancestor_cgroup_id(struct sock *sk,
4472 int ancestor_level)
4473{
4474 struct cgroup *ancestor;
4475 struct cgroup *cgrp;
4476
4477 sk = sk_to_full_sk(sk);
4478 if (!sk || !sk_fullsock(sk))
4479 return 0;
4480
4481 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
4482 ancestor = cgroup_ancestor(cgrp, ancestor_level);
4483 if (!ancestor)
4484 return 0;
4485
4486 return cgroup_id(ancestor);
4487}
4488
4489BPF_CALL_2(bpf_skb_ancestor_cgroup_id, const struct sk_buff *, skb, int,
4490 ancestor_level)
4491{
4492 return __bpf_sk_ancestor_cgroup_id(skb->sk, ancestor_level);
4493}
4494
4495static const struct bpf_func_proto bpf_skb_ancestor_cgroup_id_proto = {
4496 .func = bpf_skb_ancestor_cgroup_id,
4497 .gpl_only = false,
4498 .ret_type = RET_INTEGER,
4499 .arg1_type = ARG_PTR_TO_CTX,
4500 .arg2_type = ARG_ANYTHING,
4501};
4502
4503BPF_CALL_1(bpf_sk_cgroup_id, struct sock *, sk)
4504{
4505 return __bpf_sk_cgroup_id(sk);
4506}
4507
4508static const struct bpf_func_proto bpf_sk_cgroup_id_proto = {
4509 .func = bpf_sk_cgroup_id,
4510 .gpl_only = false,
4511 .ret_type = RET_INTEGER,
4512 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
4513};
4514
4515BPF_CALL_2(bpf_sk_ancestor_cgroup_id, struct sock *, sk, int, ancestor_level)
4516{
4517 return __bpf_sk_ancestor_cgroup_id(sk, ancestor_level);
4518}
4519
4520static const struct bpf_func_proto bpf_sk_ancestor_cgroup_id_proto = {
4521 .func = bpf_sk_ancestor_cgroup_id,
4522 .gpl_only = false,
4523 .ret_type = RET_INTEGER,
4524 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
4525 .arg2_type = ARG_ANYTHING,
4526};
4527#endif
4528
4529static unsigned long bpf_xdp_copy(void *dst_buff, const void *src_buff,
4530 unsigned long off, unsigned long len)
4531{
4532 memcpy(dst_buff, src_buff + off, len);
4533 return 0;
4534}
4535
4536BPF_CALL_5(bpf_xdp_event_output, struct xdp_buff *, xdp, struct bpf_map *, map,
4537 u64, flags, void *, meta, u64, meta_size)
4538{
4539 u64 xdp_size = (flags & BPF_F_CTXLEN_MASK) >> 32;
4540
4541 if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
4542 return -EINVAL;
4543 if (unlikely(!xdp ||
4544 xdp_size > (unsigned long)(xdp->data_end - xdp->data)))
4545 return -EFAULT;
4546
4547 return bpf_event_output(map, flags, meta, meta_size, xdp->data,
4548 xdp_size, bpf_xdp_copy);
4549}
4550
4551static const struct bpf_func_proto bpf_xdp_event_output_proto = {
4552 .func = bpf_xdp_event_output,
4553 .gpl_only = true,
4554 .ret_type = RET_INTEGER,
4555 .arg1_type = ARG_PTR_TO_CTX,
4556 .arg2_type = ARG_CONST_MAP_PTR,
4557 .arg3_type = ARG_ANYTHING,
4558 .arg4_type = ARG_PTR_TO_MEM,
4559 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
4560};
4561
4562BTF_ID_LIST_SINGLE(bpf_xdp_output_btf_ids, struct, xdp_buff)
4563
4564const struct bpf_func_proto bpf_xdp_output_proto = {
4565 .func = bpf_xdp_event_output,
4566 .gpl_only = true,
4567 .ret_type = RET_INTEGER,
4568 .arg1_type = ARG_PTR_TO_BTF_ID,
4569 .arg1_btf_id = &bpf_xdp_output_btf_ids[0],
4570 .arg2_type = ARG_CONST_MAP_PTR,
4571 .arg3_type = ARG_ANYTHING,
4572 .arg4_type = ARG_PTR_TO_MEM,
4573 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
4574};
4575
4576BPF_CALL_1(bpf_get_socket_cookie, struct sk_buff *, skb)
4577{
4578 return skb->sk ? __sock_gen_cookie(skb->sk) : 0;
4579}
4580
4581static const struct bpf_func_proto bpf_get_socket_cookie_proto = {
4582 .func = bpf_get_socket_cookie,
4583 .gpl_only = false,
4584 .ret_type = RET_INTEGER,
4585 .arg1_type = ARG_PTR_TO_CTX,
4586};
4587
4588BPF_CALL_1(bpf_get_socket_cookie_sock_addr, struct bpf_sock_addr_kern *, ctx)
4589{
4590 return __sock_gen_cookie(ctx->sk);
4591}
4592
4593static const struct bpf_func_proto bpf_get_socket_cookie_sock_addr_proto = {
4594 .func = bpf_get_socket_cookie_sock_addr,
4595 .gpl_only = false,
4596 .ret_type = RET_INTEGER,
4597 .arg1_type = ARG_PTR_TO_CTX,
4598};
4599
4600BPF_CALL_1(bpf_get_socket_cookie_sock, struct sock *, ctx)
4601{
4602 return __sock_gen_cookie(ctx);
4603}
4604
4605static const struct bpf_func_proto bpf_get_socket_cookie_sock_proto = {
4606 .func = bpf_get_socket_cookie_sock,
4607 .gpl_only = false,
4608 .ret_type = RET_INTEGER,
4609 .arg1_type = ARG_PTR_TO_CTX,
4610};
4611
4612BPF_CALL_1(bpf_get_socket_ptr_cookie, struct sock *, sk)
4613{
4614 return sk ? sock_gen_cookie(sk) : 0;
4615}
4616
4617const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto = {
4618 .func = bpf_get_socket_ptr_cookie,
4619 .gpl_only = false,
4620 .ret_type = RET_INTEGER,
4621 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
4622};
4623
4624BPF_CALL_1(bpf_get_socket_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)
4625{
4626 return __sock_gen_cookie(ctx->sk);
4627}
4628
4629static const struct bpf_func_proto bpf_get_socket_cookie_sock_ops_proto = {
4630 .func = bpf_get_socket_cookie_sock_ops,
4631 .gpl_only = false,
4632 .ret_type = RET_INTEGER,
4633 .arg1_type = ARG_PTR_TO_CTX,
4634};
4635
4636static u64 __bpf_get_netns_cookie(struct sock *sk)
4637{
4638 const struct net *net = sk ? sock_net(sk) : &init_net;
4639
4640 return net->net_cookie;
4641}
4642
4643BPF_CALL_1(bpf_get_netns_cookie_sock, struct sock *, ctx)
4644{
4645 return __bpf_get_netns_cookie(ctx);
4646}
4647
4648static const struct bpf_func_proto bpf_get_netns_cookie_sock_proto = {
4649 .func = bpf_get_netns_cookie_sock,
4650 .gpl_only = false,
4651 .ret_type = RET_INTEGER,
4652 .arg1_type = ARG_PTR_TO_CTX_OR_NULL,
4653};
4654
4655BPF_CALL_1(bpf_get_netns_cookie_sock_addr, struct bpf_sock_addr_kern *, ctx)
4656{
4657 return __bpf_get_netns_cookie(ctx ? ctx->sk : NULL);
4658}
4659
4660static const struct bpf_func_proto bpf_get_netns_cookie_sock_addr_proto = {
4661 .func = bpf_get_netns_cookie_sock_addr,
4662 .gpl_only = false,
4663 .ret_type = RET_INTEGER,
4664 .arg1_type = ARG_PTR_TO_CTX_OR_NULL,
4665};
4666
4667BPF_CALL_1(bpf_get_socket_uid, struct sk_buff *, skb)
4668{
4669 struct sock *sk = sk_to_full_sk(skb->sk);
4670 kuid_t kuid;
4671
4672 if (!sk || !sk_fullsock(sk))
4673 return overflowuid;
4674 kuid = sock_net_uid(sock_net(sk), sk);
4675 return from_kuid_munged(sock_net(sk)->user_ns, kuid);
4676}
4677
4678static const struct bpf_func_proto bpf_get_socket_uid_proto = {
4679 .func = bpf_get_socket_uid,
4680 .gpl_only = false,
4681 .ret_type = RET_INTEGER,
4682 .arg1_type = ARG_PTR_TO_CTX,
4683};
4684
4685static int _bpf_setsockopt(struct sock *sk, int level, int optname,
4686 char *optval, int optlen)
4687{
4688 char devname[IFNAMSIZ];
4689 int val, valbool;
4690 struct net *net;
4691 int ifindex;
4692 int ret = 0;
4693
4694 if (!sk_fullsock(sk))
4695 return -EINVAL;
4696
4697 sock_owned_by_me(sk);
4698
4699 if (level == SOL_SOCKET) {
4700 if (optlen != sizeof(int) && optname != SO_BINDTODEVICE)
4701 return -EINVAL;
4702 val = *((int *)optval);
4703 valbool = val ? 1 : 0;
4704
4705
4706 switch (optname) {
4707 case SO_RCVBUF:
4708 val = min_t(u32, val, sysctl_rmem_max);
4709 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
4710 WRITE_ONCE(sk->sk_rcvbuf,
4711 max_t(int, val * 2, SOCK_MIN_RCVBUF));
4712 break;
4713 case SO_SNDBUF:
4714 val = min_t(u32, val, sysctl_wmem_max);
4715 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
4716 WRITE_ONCE(sk->sk_sndbuf,
4717 max_t(int, val * 2, SOCK_MIN_SNDBUF));
4718 break;
4719 case SO_MAX_PACING_RATE:
4720 if (val != ~0U)
4721 cmpxchg(&sk->sk_pacing_status,
4722 SK_PACING_NONE,
4723 SK_PACING_NEEDED);
4724 sk->sk_max_pacing_rate = (val == ~0U) ?
4725 ~0UL : (unsigned int)val;
4726 sk->sk_pacing_rate = min(sk->sk_pacing_rate,
4727 sk->sk_max_pacing_rate);
4728 break;
4729 case SO_PRIORITY:
4730 sk->sk_priority = val;
4731 break;
4732 case SO_RCVLOWAT:
4733 if (val < 0)
4734 val = INT_MAX;
4735 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
4736 break;
4737 case SO_MARK:
4738 if (sk->sk_mark != val) {
4739 sk->sk_mark = val;
4740 sk_dst_reset(sk);
4741 }
4742 break;
4743 case SO_BINDTODEVICE:
4744 optlen = min_t(long, optlen, IFNAMSIZ - 1);
4745 strncpy(devname, optval, optlen);
4746 devname[optlen] = 0;
4747
4748 ifindex = 0;
4749 if (devname[0] != '\0') {
4750 struct net_device *dev;
4751
4752 ret = -ENODEV;
4753
4754 net = sock_net(sk);
4755 dev = dev_get_by_name(net, devname);
4756 if (!dev)
4757 break;
4758 ifindex = dev->ifindex;
4759 dev_put(dev);
4760 }
4761 fallthrough;
4762 case SO_BINDTOIFINDEX:
4763 if (optname == SO_BINDTOIFINDEX)
4764 ifindex = val;
4765 ret = sock_bindtoindex(sk, ifindex, false);
4766 break;
4767 case SO_KEEPALIVE:
4768 if (sk->sk_prot->keepalive)
4769 sk->sk_prot->keepalive(sk, valbool);
4770 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
4771 break;
4772 case SO_REUSEPORT:
4773 sk->sk_reuseport = valbool;
4774 break;
4775 default:
4776 ret = -EINVAL;
4777 }
4778#ifdef CONFIG_INET
4779 } else if (level == SOL_IP) {
4780 if (optlen != sizeof(int) || sk->sk_family != AF_INET)
4781 return -EINVAL;
4782
4783 val = *((int *)optval);
4784
4785 switch (optname) {
4786 case IP_TOS:
4787 if (val < -1 || val > 0xff) {
4788 ret = -EINVAL;
4789 } else {
4790 struct inet_sock *inet = inet_sk(sk);
4791
4792 if (val == -1)
4793 val = 0;
4794 inet->tos = val;
4795 }
4796 break;
4797 default:
4798 ret = -EINVAL;
4799 }
4800#if IS_ENABLED(CONFIG_IPV6)
4801 } else if (level == SOL_IPV6) {
4802 if (optlen != sizeof(int) || sk->sk_family != AF_INET6)
4803 return -EINVAL;
4804
4805 val = *((int *)optval);
4806
4807 switch (optname) {
4808 case IPV6_TCLASS:
4809 if (val < -1 || val > 0xff) {
4810 ret = -EINVAL;
4811 } else {
4812 struct ipv6_pinfo *np = inet6_sk(sk);
4813
4814 if (val == -1)
4815 val = 0;
4816 np->tclass = val;
4817 }
4818 break;
4819 default:
4820 ret = -EINVAL;
4821 }
4822#endif
4823 } else if (level == SOL_TCP &&
4824 sk->sk_prot->setsockopt == tcp_setsockopt) {
4825 if (optname == TCP_CONGESTION) {
4826 char name[TCP_CA_NAME_MAX];
4827
4828 strncpy(name, optval, min_t(long, optlen,
4829 TCP_CA_NAME_MAX-1));
4830 name[TCP_CA_NAME_MAX-1] = 0;
4831 ret = tcp_set_congestion_control(sk, name, false, true);
4832 } else {
4833 struct inet_connection_sock *icsk = inet_csk(sk);
4834 struct tcp_sock *tp = tcp_sk(sk);
4835 unsigned long timeout;
4836
4837 if (optlen != sizeof(int))
4838 return -EINVAL;
4839
4840 val = *((int *)optval);
4841
4842 switch (optname) {
4843 case TCP_BPF_IW:
4844 if (val <= 0 || tp->data_segs_out > tp->syn_data)
4845 ret = -EINVAL;
4846 else
4847 tp->snd_cwnd = val;
4848 break;
4849 case TCP_BPF_SNDCWND_CLAMP:
4850 if (val <= 0) {
4851 ret = -EINVAL;
4852 } else {
4853 tp->snd_cwnd_clamp = val;
4854 tp->snd_ssthresh = val;
4855 }
4856 break;
4857 case TCP_BPF_DELACK_MAX:
4858 timeout = usecs_to_jiffies(val);
4859 if (timeout > TCP_DELACK_MAX ||
4860 timeout < TCP_TIMEOUT_MIN)
4861 return -EINVAL;
4862 inet_csk(sk)->icsk_delack_max = timeout;
4863 break;
4864 case TCP_BPF_RTO_MIN:
4865 timeout = usecs_to_jiffies(val);
4866 if (timeout > TCP_RTO_MIN ||
4867 timeout < TCP_TIMEOUT_MIN)
4868 return -EINVAL;
4869 inet_csk(sk)->icsk_rto_min = timeout;
4870 break;
4871 case TCP_SAVE_SYN:
4872 if (val < 0 || val > 1)
4873 ret = -EINVAL;
4874 else
4875 tp->save_syn = val;
4876 break;
4877 case TCP_KEEPIDLE:
4878 ret = tcp_sock_set_keepidle_locked(sk, val);
4879 break;
4880 case TCP_KEEPINTVL:
4881 if (val < 1 || val > MAX_TCP_KEEPINTVL)
4882 ret = -EINVAL;
4883 else
4884 tp->keepalive_intvl = val * HZ;
4885 break;
4886 case TCP_KEEPCNT:
4887 if (val < 1 || val > MAX_TCP_KEEPCNT)
4888 ret = -EINVAL;
4889 else
4890 tp->keepalive_probes = val;
4891 break;
4892 case TCP_SYNCNT:
4893 if (val < 1 || val > MAX_TCP_SYNCNT)
4894 ret = -EINVAL;
4895 else
4896 icsk->icsk_syn_retries = val;
4897 break;
4898 case TCP_USER_TIMEOUT:
4899 if (val < 0)
4900 ret = -EINVAL;
4901 else
4902 icsk->icsk_user_timeout = val;
4903 break;
4904 case TCP_NOTSENT_LOWAT:
4905 tp->notsent_lowat = val;
4906 sk->sk_write_space(sk);
4907 break;
4908 case TCP_WINDOW_CLAMP:
4909 ret = tcp_set_window_clamp(sk, val);
4910 break;
4911 default:
4912 ret = -EINVAL;
4913 }
4914 }
4915#endif
4916 } else {
4917 ret = -EINVAL;
4918 }
4919 return ret;
4920}
4921
4922static int _bpf_getsockopt(struct sock *sk, int level, int optname,
4923 char *optval, int optlen)
4924{
4925 if (!sk_fullsock(sk))
4926 goto err_clear;
4927
4928 sock_owned_by_me(sk);
4929
4930 if (level == SOL_SOCKET) {
4931 if (optlen != sizeof(int))
4932 goto err_clear;
4933
4934 switch (optname) {
4935 case SO_MARK:
4936 *((int *)optval) = sk->sk_mark;
4937 break;
4938 case SO_PRIORITY:
4939 *((int *)optval) = sk->sk_priority;
4940 break;
4941 case SO_BINDTOIFINDEX:
4942 *((int *)optval) = sk->sk_bound_dev_if;
4943 break;
4944 case SO_REUSEPORT:
4945 *((int *)optval) = sk->sk_reuseport;
4946 break;
4947 default:
4948 goto err_clear;
4949 }
4950#ifdef CONFIG_INET
4951 } else if (level == SOL_TCP && sk->sk_prot->getsockopt == tcp_getsockopt) {
4952 struct inet_connection_sock *icsk;
4953 struct tcp_sock *tp;
4954
4955 switch (optname) {
4956 case TCP_CONGESTION:
4957 icsk = inet_csk(sk);
4958
4959 if (!icsk->icsk_ca_ops || optlen <= 1)
4960 goto err_clear;
4961 strncpy(optval, icsk->icsk_ca_ops->name, optlen);
4962 optval[optlen - 1] = 0;
4963 break;
4964 case TCP_SAVED_SYN:
4965 tp = tcp_sk(sk);
4966
4967 if (optlen <= 0 || !tp->saved_syn ||
4968 optlen > tcp_saved_syn_len(tp->saved_syn))
4969 goto err_clear;
4970 memcpy(optval, tp->saved_syn->data, optlen);
4971 break;
4972 default:
4973 goto err_clear;
4974 }
4975 } else if (level == SOL_IP) {
4976 struct inet_sock *inet = inet_sk(sk);
4977
4978 if (optlen != sizeof(int) || sk->sk_family != AF_INET)
4979 goto err_clear;
4980
4981
4982 switch (optname) {
4983 case IP_TOS:
4984 *((int *)optval) = (int)inet->tos;
4985 break;
4986 default:
4987 goto err_clear;
4988 }
4989#if IS_ENABLED(CONFIG_IPV6)
4990 } else if (level == SOL_IPV6) {
4991 struct ipv6_pinfo *np = inet6_sk(sk);
4992
4993 if (optlen != sizeof(int) || sk->sk_family != AF_INET6)
4994 goto err_clear;
4995
4996
4997 switch (optname) {
4998 case IPV6_TCLASS:
4999 *((int *)optval) = (int)np->tclass;
5000 break;
5001 default:
5002 goto err_clear;
5003 }
5004#endif
5005#endif
5006 } else {
5007 goto err_clear;
5008 }
5009 return 0;
5010err_clear:
5011 memset(optval, 0, optlen);
5012 return -EINVAL;
5013}
5014
5015BPF_CALL_5(bpf_sock_addr_setsockopt, struct bpf_sock_addr_kern *, ctx,
5016 int, level, int, optname, char *, optval, int, optlen)
5017{
5018 return _bpf_setsockopt(ctx->sk, level, optname, optval, optlen);
5019}
5020
5021static const struct bpf_func_proto bpf_sock_addr_setsockopt_proto = {
5022 .func = bpf_sock_addr_setsockopt,
5023 .gpl_only = false,
5024 .ret_type = RET_INTEGER,
5025 .arg1_type = ARG_PTR_TO_CTX,
5026 .arg2_type = ARG_ANYTHING,
5027 .arg3_type = ARG_ANYTHING,
5028 .arg4_type = ARG_PTR_TO_MEM,
5029 .arg5_type = ARG_CONST_SIZE,
5030};
5031
5032BPF_CALL_5(bpf_sock_addr_getsockopt, struct bpf_sock_addr_kern *, ctx,
5033 int, level, int, optname, char *, optval, int, optlen)
5034{
5035 return _bpf_getsockopt(ctx->sk, level, optname, optval, optlen);
5036}
5037
5038static const struct bpf_func_proto bpf_sock_addr_getsockopt_proto = {
5039 .func = bpf_sock_addr_getsockopt,
5040 .gpl_only = false,
5041 .ret_type = RET_INTEGER,
5042 .arg1_type = ARG_PTR_TO_CTX,
5043 .arg2_type = ARG_ANYTHING,
5044 .arg3_type = ARG_ANYTHING,
5045 .arg4_type = ARG_PTR_TO_UNINIT_MEM,
5046 .arg5_type = ARG_CONST_SIZE,
5047};
5048
5049BPF_CALL_5(bpf_sock_ops_setsockopt, struct bpf_sock_ops_kern *, bpf_sock,
5050 int, level, int, optname, char *, optval, int, optlen)
5051{
5052 return _bpf_setsockopt(bpf_sock->sk, level, optname, optval, optlen);
5053}
5054
5055static const struct bpf_func_proto bpf_sock_ops_setsockopt_proto = {
5056 .func = bpf_sock_ops_setsockopt,
5057 .gpl_only = false,
5058 .ret_type = RET_INTEGER,
5059 .arg1_type = ARG_PTR_TO_CTX,
5060 .arg2_type = ARG_ANYTHING,
5061 .arg3_type = ARG_ANYTHING,
5062 .arg4_type = ARG_PTR_TO_MEM,
5063 .arg5_type = ARG_CONST_SIZE,
5064};
5065
5066static int bpf_sock_ops_get_syn(struct bpf_sock_ops_kern *bpf_sock,
5067 int optname, const u8 **start)
5068{
5069 struct sk_buff *syn_skb = bpf_sock->syn_skb;
5070 const u8 *hdr_start;
5071 int ret;
5072
5073 if (syn_skb) {
5074
5075
5076 if (optname == TCP_BPF_SYN) {
5077 hdr_start = syn_skb->data;
5078 ret = tcp_hdrlen(syn_skb);
5079 } else if (optname == TCP_BPF_SYN_IP) {
5080 hdr_start = skb_network_header(syn_skb);
5081 ret = skb_network_header_len(syn_skb) +
5082 tcp_hdrlen(syn_skb);
5083 } else {
5084
5085 hdr_start = skb_mac_header(syn_skb);
5086 ret = skb_mac_header_len(syn_skb) +
5087 skb_network_header_len(syn_skb) +
5088 tcp_hdrlen(syn_skb);
5089 }
5090 } else {
5091 struct sock *sk = bpf_sock->sk;
5092 struct saved_syn *saved_syn;
5093
5094 if (sk->sk_state == TCP_NEW_SYN_RECV)
5095
5096
5097
5098
5099 saved_syn = inet_reqsk(sk)->saved_syn;
5100 else
5101 saved_syn = tcp_sk(sk)->saved_syn;
5102
5103 if (!saved_syn)
5104 return -ENOENT;
5105
5106 if (optname == TCP_BPF_SYN) {
5107 hdr_start = saved_syn->data +
5108 saved_syn->mac_hdrlen +
5109 saved_syn->network_hdrlen;
5110 ret = saved_syn->tcp_hdrlen;
5111 } else if (optname == TCP_BPF_SYN_IP) {
5112 hdr_start = saved_syn->data +
5113 saved_syn->mac_hdrlen;
5114 ret = saved_syn->network_hdrlen +
5115 saved_syn->tcp_hdrlen;
5116 } else {
5117
5118
5119
5120 if (!saved_syn->mac_hdrlen)
5121 return -ENOENT;
5122
5123 hdr_start = saved_syn->data;
5124 ret = saved_syn->mac_hdrlen +
5125 saved_syn->network_hdrlen +
5126 saved_syn->tcp_hdrlen;
5127 }
5128 }
5129
5130 *start = hdr_start;
5131 return ret;
5132}
5133
5134BPF_CALL_5(bpf_sock_ops_getsockopt, struct bpf_sock_ops_kern *, bpf_sock,
5135 int, level, int, optname, char *, optval, int, optlen)
5136{
5137 if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP &&
5138 optname >= TCP_BPF_SYN && optname <= TCP_BPF_SYN_MAC) {
5139 int ret, copy_len = 0;
5140 const u8 *start;
5141
5142 ret = bpf_sock_ops_get_syn(bpf_sock, optname, &start);
5143 if (ret > 0) {
5144 copy_len = ret;
5145 if (optlen < copy_len) {
5146 copy_len = optlen;
5147 ret = -ENOSPC;
5148 }
5149
5150 memcpy(optval, start, copy_len);
5151 }
5152
5153
5154 memset(optval + copy_len, 0, optlen - copy_len);
5155
5156 return ret;
5157 }
5158
5159 return _bpf_getsockopt(bpf_sock->sk, level, optname, optval, optlen);
5160}
5161
5162static const struct bpf_func_proto bpf_sock_ops_getsockopt_proto = {
5163 .func = bpf_sock_ops_getsockopt,
5164 .gpl_only = false,
5165 .ret_type = RET_INTEGER,
5166 .arg1_type = ARG_PTR_TO_CTX,
5167 .arg2_type = ARG_ANYTHING,
5168 .arg3_type = ARG_ANYTHING,
5169 .arg4_type = ARG_PTR_TO_UNINIT_MEM,
5170 .arg5_type = ARG_CONST_SIZE,
5171};
5172
5173BPF_CALL_2(bpf_sock_ops_cb_flags_set, struct bpf_sock_ops_kern *, bpf_sock,
5174 int, argval)
5175{
5176 struct sock *sk = bpf_sock->sk;
5177 int val = argval & BPF_SOCK_OPS_ALL_CB_FLAGS;
5178
5179 if (!IS_ENABLED(CONFIG_INET) || !sk_fullsock(sk))
5180 return -EINVAL;
5181
5182 tcp_sk(sk)->bpf_sock_ops_cb_flags = val;
5183
5184 return argval & (~BPF_SOCK_OPS_ALL_CB_FLAGS);
5185}
5186
5187static const struct bpf_func_proto bpf_sock_ops_cb_flags_set_proto = {
5188 .func = bpf_sock_ops_cb_flags_set,
5189 .gpl_only = false,
5190 .ret_type = RET_INTEGER,
5191 .arg1_type = ARG_PTR_TO_CTX,
5192 .arg2_type = ARG_ANYTHING,
5193};
5194
5195const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly;
5196EXPORT_SYMBOL_GPL(ipv6_bpf_stub);
5197
5198BPF_CALL_3(bpf_bind, struct bpf_sock_addr_kern *, ctx, struct sockaddr *, addr,
5199 int, addr_len)
5200{
5201#ifdef CONFIG_INET
5202 struct sock *sk = ctx->sk;
5203 u32 flags = BIND_FROM_BPF;
5204 int err;
5205
5206 err = -EINVAL;
5207 if (addr_len < offsetofend(struct sockaddr, sa_family))
5208 return err;
5209 if (addr->sa_family == AF_INET) {
5210 if (addr_len < sizeof(struct sockaddr_in))
5211 return err;
5212 if (((struct sockaddr_in *)addr)->sin_port == htons(0))
5213 flags |= BIND_FORCE_ADDRESS_NO_PORT;
5214 return __inet_bind(sk, addr, addr_len, flags);
5215#if IS_ENABLED(CONFIG_IPV6)
5216 } else if (addr->sa_family == AF_INET6) {
5217 if (addr_len < SIN6_LEN_RFC2133)
5218 return err;
5219 if (((struct sockaddr_in6 *)addr)->sin6_port == htons(0))
5220 flags |= BIND_FORCE_ADDRESS_NO_PORT;
5221
5222
5223
5224 return ipv6_bpf_stub->inet6_bind(sk, addr, addr_len, flags);
5225#endif
5226 }
5227#endif
5228
5229 return -EAFNOSUPPORT;
5230}
5231
5232static const struct bpf_func_proto bpf_bind_proto = {
5233 .func = bpf_bind,
5234 .gpl_only = false,
5235 .ret_type = RET_INTEGER,
5236 .arg1_type = ARG_PTR_TO_CTX,
5237 .arg2_type = ARG_PTR_TO_MEM,
5238 .arg3_type = ARG_CONST_SIZE,
5239};
5240
5241#ifdef CONFIG_XFRM
5242BPF_CALL_5(bpf_skb_get_xfrm_state, struct sk_buff *, skb, u32, index,
5243 struct bpf_xfrm_state *, to, u32, size, u64, flags)
5244{
5245 const struct sec_path *sp = skb_sec_path(skb);
5246 const struct xfrm_state *x;
5247
5248 if (!sp || unlikely(index >= sp->len || flags))
5249 goto err_clear;
5250
5251 x = sp->xvec[index];
5252
5253 if (unlikely(size != sizeof(struct bpf_xfrm_state)))
5254 goto err_clear;
5255
5256 to->reqid = x->props.reqid;
5257 to->spi = x->id.spi;
5258 to->family = x->props.family;
5259 to->ext = 0;
5260
5261 if (to->family == AF_INET6) {
5262 memcpy(to->remote_ipv6, x->props.saddr.a6,
5263 sizeof(to->remote_ipv6));
5264 } else {
5265 to->remote_ipv4 = x->props.saddr.a4;
5266 memset(&to->remote_ipv6[1], 0, sizeof(__u32) * 3);
5267 }
5268
5269 return 0;
5270err_clear:
5271 memset(to, 0, size);
5272 return -EINVAL;
5273}
5274
5275static const struct bpf_func_proto bpf_skb_get_xfrm_state_proto = {
5276 .func = bpf_skb_get_xfrm_state,
5277 .gpl_only = false,
5278 .ret_type = RET_INTEGER,
5279 .arg1_type = ARG_PTR_TO_CTX,
5280 .arg2_type = ARG_ANYTHING,
5281 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
5282 .arg4_type = ARG_CONST_SIZE,
5283 .arg5_type = ARG_ANYTHING,
5284};
5285#endif
5286
5287#if IS_ENABLED(CONFIG_INET) || IS_ENABLED(CONFIG_IPV6)
5288static int bpf_fib_set_fwd_params(struct bpf_fib_lookup *params,
5289 const struct neighbour *neigh,
5290 const struct net_device *dev, u32 mtu)
5291{
5292 memcpy(params->dmac, neigh->ha, ETH_ALEN);
5293 memcpy(params->smac, dev->dev_addr, ETH_ALEN);
5294 params->h_vlan_TCI = 0;
5295 params->h_vlan_proto = 0;
5296 if (mtu)
5297 params->mtu_result = mtu;
5298
5299 return 0;
5300}
5301#endif
5302
5303#if IS_ENABLED(CONFIG_INET)
5304static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
5305 u32 flags, bool check_mtu)
5306{
5307 struct fib_nh_common *nhc;
5308 struct in_device *in_dev;
5309 struct neighbour *neigh;
5310 struct net_device *dev;
5311 struct fib_result res;
5312 struct flowi4 fl4;
5313 u32 mtu = 0;
5314 int err;
5315
5316 dev = dev_get_by_index_rcu(net, params->ifindex);
5317 if (unlikely(!dev))
5318 return -ENODEV;
5319
5320
5321 in_dev = __in_dev_get_rcu(dev);
5322 if (unlikely(!in_dev || !IN_DEV_FORWARD(in_dev)))
5323 return BPF_FIB_LKUP_RET_FWD_DISABLED;
5324
5325 if (flags & BPF_FIB_LOOKUP_OUTPUT) {
5326 fl4.flowi4_iif = 1;
5327 fl4.flowi4_oif = params->ifindex;
5328 } else {
5329 fl4.flowi4_iif = params->ifindex;
5330 fl4.flowi4_oif = 0;
5331 }
5332 fl4.flowi4_tos = params->tos & IPTOS_RT_MASK;
5333 fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
5334 fl4.flowi4_flags = 0;
5335
5336 fl4.flowi4_proto = params->l4_protocol;
5337 fl4.daddr = params->ipv4_dst;
5338 fl4.saddr = params->ipv4_src;
5339 fl4.fl4_sport = params->sport;
5340 fl4.fl4_dport = params->dport;
5341 fl4.flowi4_multipath_hash = 0;
5342
5343 if (flags & BPF_FIB_LOOKUP_DIRECT) {
5344 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
5345 struct fib_table *tb;
5346
5347 tb = fib_get_table(net, tbid);
5348 if (unlikely(!tb))
5349 return BPF_FIB_LKUP_RET_NOT_FWDED;
5350
5351 err = fib_table_lookup(tb, &fl4, &res, FIB_LOOKUP_NOREF);
5352 } else {
5353 fl4.flowi4_mark = 0;
5354 fl4.flowi4_secid = 0;
5355 fl4.flowi4_tun_key.tun_id = 0;
5356 fl4.flowi4_uid = sock_net_uid(net, NULL);
5357
5358 err = fib_lookup(net, &fl4, &res, FIB_LOOKUP_NOREF);
5359 }
5360
5361 if (err) {
5362
5363 if (err == -EINVAL)
5364 return BPF_FIB_LKUP_RET_BLACKHOLE;
5365 if (err == -EHOSTUNREACH)
5366 return BPF_FIB_LKUP_RET_UNREACHABLE;
5367 if (err == -EACCES)
5368 return BPF_FIB_LKUP_RET_PROHIBIT;
5369
5370 return BPF_FIB_LKUP_RET_NOT_FWDED;
5371 }
5372
5373 if (res.type != RTN_UNICAST)
5374 return BPF_FIB_LKUP_RET_NOT_FWDED;
5375
5376 if (fib_info_num_path(res.fi) > 1)
5377 fib_select_path(net, &res, &fl4, NULL);
5378
5379 if (check_mtu) {
5380 mtu = ip_mtu_from_fib_result(&res, params->ipv4_dst);
5381 if (params->tot_len > mtu) {
5382 params->mtu_result = mtu;
5383 return BPF_FIB_LKUP_RET_FRAG_NEEDED;
5384 }
5385 }
5386
5387 nhc = res.nhc;
5388
5389
5390 if (nhc->nhc_lwtstate)
5391 return BPF_FIB_LKUP_RET_UNSUPP_LWT;
5392
5393 dev = nhc->nhc_dev;
5394
5395 params->rt_metric = res.fi->fib_priority;
5396 params->ifindex = dev->ifindex;
5397
5398
5399
5400
5401 if (likely(nhc->nhc_gw_family != AF_INET6)) {
5402 if (nhc->nhc_gw_family)
5403 params->ipv4_dst = nhc->nhc_gw.ipv4;
5404
5405 neigh = __ipv4_neigh_lookup_noref(dev,
5406 (__force u32)params->ipv4_dst);
5407 } else {
5408 struct in6_addr *dst = (struct in6_addr *)params->ipv6_dst;
5409
5410 params->family = AF_INET6;
5411 *dst = nhc->nhc_gw.ipv6;
5412 neigh = __ipv6_neigh_lookup_noref_stub(dev, dst);
5413 }
5414
5415 if (!neigh)
5416 return BPF_FIB_LKUP_RET_NO_NEIGH;
5417
5418 return bpf_fib_set_fwd_params(params, neigh, dev, mtu);
5419}
5420#endif
5421
5422#if IS_ENABLED(CONFIG_IPV6)
5423static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
5424 u32 flags, bool check_mtu)
5425{
5426 struct in6_addr *src = (struct in6_addr *) params->ipv6_src;
5427 struct in6_addr *dst = (struct in6_addr *) params->ipv6_dst;
5428 struct fib6_result res = {};
5429 struct neighbour *neigh;
5430 struct net_device *dev;
5431 struct inet6_dev *idev;
5432 struct flowi6 fl6;
5433 int strict = 0;
5434 int oif, err;
5435 u32 mtu = 0;
5436
5437
5438 if (rt6_need_strict(dst) || rt6_need_strict(src))
5439 return BPF_FIB_LKUP_RET_NOT_FWDED;
5440
5441 dev = dev_get_by_index_rcu(net, params->ifindex);
5442 if (unlikely(!dev))
5443 return -ENODEV;
5444
5445 idev = __in6_dev_get_safely(dev);
5446 if (unlikely(!idev || !idev->cnf.forwarding))
5447 return BPF_FIB_LKUP_RET_FWD_DISABLED;
5448
5449 if (flags & BPF_FIB_LOOKUP_OUTPUT) {
5450 fl6.flowi6_iif = 1;
5451 oif = fl6.flowi6_oif = params->ifindex;
5452 } else {
5453 oif = fl6.flowi6_iif = params->ifindex;
5454 fl6.flowi6_oif = 0;
5455 strict = RT6_LOOKUP_F_HAS_SADDR;
5456 }
5457 fl6.flowlabel = params->flowinfo;
5458 fl6.flowi6_scope = 0;
5459 fl6.flowi6_flags = 0;
5460 fl6.mp_hash = 0;
5461
5462 fl6.flowi6_proto = params->l4_protocol;
5463 fl6.daddr = *dst;
5464 fl6.saddr = *src;
5465 fl6.fl6_sport = params->sport;
5466 fl6.fl6_dport = params->dport;
5467
5468 if (flags & BPF_FIB_LOOKUP_DIRECT) {
5469 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
5470 struct fib6_table *tb;
5471
5472 tb = ipv6_stub->fib6_get_table(net, tbid);
5473 if (unlikely(!tb))
5474 return BPF_FIB_LKUP_RET_NOT_FWDED;
5475
5476 err = ipv6_stub->fib6_table_lookup(net, tb, oif, &fl6, &res,
5477 strict);
5478 } else {
5479 fl6.flowi6_mark = 0;
5480 fl6.flowi6_secid = 0;
5481 fl6.flowi6_tun_key.tun_id = 0;
5482 fl6.flowi6_uid = sock_net_uid(net, NULL);
5483
5484 err = ipv6_stub->fib6_lookup(net, oif, &fl6, &res, strict);
5485 }
5486
5487 if (unlikely(err || IS_ERR_OR_NULL(res.f6i) ||
5488 res.f6i == net->ipv6.fib6_null_entry))
5489 return BPF_FIB_LKUP_RET_NOT_FWDED;
5490
5491 switch (res.fib6_type) {
5492
5493 case RTN_UNICAST:
5494 break;
5495 case RTN_BLACKHOLE:
5496 return BPF_FIB_LKUP_RET_BLACKHOLE;
5497 case RTN_UNREACHABLE:
5498 return BPF_FIB_LKUP_RET_UNREACHABLE;
5499 case RTN_PROHIBIT:
5500 return BPF_FIB_LKUP_RET_PROHIBIT;
5501 default:
5502 return BPF_FIB_LKUP_RET_NOT_FWDED;
5503 }
5504
5505 ipv6_stub->fib6_select_path(net, &res, &fl6, fl6.flowi6_oif,
5506 fl6.flowi6_oif != 0, NULL, strict);
5507
5508 if (check_mtu) {
5509 mtu = ipv6_stub->ip6_mtu_from_fib6(&res, dst, src);
5510 if (params->tot_len > mtu) {
5511 params->mtu_result = mtu;
5512 return BPF_FIB_LKUP_RET_FRAG_NEEDED;
5513 }
5514 }
5515
5516 if (res.nh->fib_nh_lws)
5517 return BPF_FIB_LKUP_RET_UNSUPP_LWT;
5518
5519 if (res.nh->fib_nh_gw_family)
5520 *dst = res.nh->fib_nh_gw6;
5521
5522 dev = res.nh->fib_nh_dev;
5523 params->rt_metric = res.f6i->fib6_metric;
5524 params->ifindex = dev->ifindex;
5525
5526
5527
5528
5529 neigh = __ipv6_neigh_lookup_noref_stub(dev, dst);
5530 if (!neigh)
5531 return BPF_FIB_LKUP_RET_NO_NEIGH;
5532
5533 return bpf_fib_set_fwd_params(params, neigh, dev, mtu);
5534}
5535#endif
5536
5537BPF_CALL_4(bpf_xdp_fib_lookup, struct xdp_buff *, ctx,
5538 struct bpf_fib_lookup *, params, int, plen, u32, flags)
5539{
5540 if (plen < sizeof(*params))
5541 return -EINVAL;
5542
5543 if (flags & ~(BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT))
5544 return -EINVAL;
5545
5546 switch (params->family) {
5547#if IS_ENABLED(CONFIG_INET)
5548 case AF_INET:
5549 return bpf_ipv4_fib_lookup(dev_net(ctx->rxq->dev), params,
5550 flags, true);
5551#endif
5552#if IS_ENABLED(CONFIG_IPV6)
5553 case AF_INET6:
5554 return bpf_ipv6_fib_lookup(dev_net(ctx->rxq->dev), params,
5555 flags, true);
5556#endif
5557 }
5558 return -EAFNOSUPPORT;
5559}
5560
5561static const struct bpf_func_proto bpf_xdp_fib_lookup_proto = {
5562 .func = bpf_xdp_fib_lookup,
5563 .gpl_only = true,
5564 .ret_type = RET_INTEGER,
5565 .arg1_type = ARG_PTR_TO_CTX,
5566 .arg2_type = ARG_PTR_TO_MEM,
5567 .arg3_type = ARG_CONST_SIZE,
5568 .arg4_type = ARG_ANYTHING,
5569};
5570
5571BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb,
5572 struct bpf_fib_lookup *, params, int, plen, u32, flags)
5573{
5574 struct net *net = dev_net(skb->dev);
5575 int rc = -EAFNOSUPPORT;
5576 bool check_mtu = false;
5577
5578 if (plen < sizeof(*params))
5579 return -EINVAL;
5580
5581 if (flags & ~(BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT))
5582 return -EINVAL;
5583
5584 if (params->tot_len)
5585 check_mtu = true;
5586
5587 switch (params->family) {
5588#if IS_ENABLED(CONFIG_INET)
5589 case AF_INET:
5590 rc = bpf_ipv4_fib_lookup(net, params, flags, check_mtu);
5591 break;
5592#endif
5593#if IS_ENABLED(CONFIG_IPV6)
5594 case AF_INET6:
5595 rc = bpf_ipv6_fib_lookup(net, params, flags, check_mtu);
5596 break;
5597#endif
5598 }
5599
5600 if (rc == BPF_FIB_LKUP_RET_SUCCESS && !check_mtu) {
5601 struct net_device *dev;
5602
5603
5604
5605
5606 dev = dev_get_by_index_rcu(net, params->ifindex);
5607 if (!is_skb_forwardable(dev, skb))
5608 rc = BPF_FIB_LKUP_RET_FRAG_NEEDED;
5609
5610 params->mtu_result = dev->mtu;
5611 }
5612
5613 return rc;
5614}
5615
5616static const struct bpf_func_proto bpf_skb_fib_lookup_proto = {
5617 .func = bpf_skb_fib_lookup,
5618 .gpl_only = true,
5619 .ret_type = RET_INTEGER,
5620 .arg1_type = ARG_PTR_TO_CTX,
5621 .arg2_type = ARG_PTR_TO_MEM,
5622 .arg3_type = ARG_CONST_SIZE,
5623 .arg4_type = ARG_ANYTHING,
5624};
5625
5626static struct net_device *__dev_via_ifindex(struct net_device *dev_curr,
5627 u32 ifindex)
5628{
5629 struct net *netns = dev_net(dev_curr);
5630
5631
5632 if (ifindex == 0)
5633 return dev_curr;
5634
5635 return dev_get_by_index_rcu(netns, ifindex);
5636}
5637
5638BPF_CALL_5(bpf_skb_check_mtu, struct sk_buff *, skb,
5639 u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
5640{
5641 int ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
5642 struct net_device *dev = skb->dev;
5643 int skb_len, dev_len;
5644 int mtu;
5645
5646 if (unlikely(flags & ~(BPF_MTU_CHK_SEGS)))
5647 return -EINVAL;
5648
5649 if (unlikely(flags & BPF_MTU_CHK_SEGS && (len_diff || *mtu_len)))
5650 return -EINVAL;
5651
5652 dev = __dev_via_ifindex(dev, ifindex);
5653 if (unlikely(!dev))
5654 return -ENODEV;
5655
5656 mtu = READ_ONCE(dev->mtu);
5657
5658 dev_len = mtu + dev->hard_header_len;
5659
5660
5661 skb_len = *mtu_len ? *mtu_len + dev->hard_header_len : skb->len;
5662
5663 skb_len += len_diff;
5664 if (skb_len <= dev_len) {
5665 ret = BPF_MTU_CHK_RET_SUCCESS;
5666 goto out;
5667 }
5668
5669
5670
5671
5672
5673 if (skb_is_gso(skb)) {
5674 ret = BPF_MTU_CHK_RET_SUCCESS;
5675
5676 if (flags & BPF_MTU_CHK_SEGS &&
5677 !skb_gso_validate_network_len(skb, mtu))
5678 ret = BPF_MTU_CHK_RET_SEGS_TOOBIG;
5679 }
5680out:
5681
5682 *mtu_len = mtu;
5683
5684 return ret;
5685}
5686
5687BPF_CALL_5(bpf_xdp_check_mtu, struct xdp_buff *, xdp,
5688 u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
5689{
5690 struct net_device *dev = xdp->rxq->dev;
5691 int xdp_len = xdp->data_end - xdp->data;
5692 int ret = BPF_MTU_CHK_RET_SUCCESS;
5693 int mtu, dev_len;
5694
5695
5696 if (unlikely(flags))
5697 return -EINVAL;
5698
5699 dev = __dev_via_ifindex(dev, ifindex);
5700 if (unlikely(!dev))
5701 return -ENODEV;
5702
5703 mtu = READ_ONCE(dev->mtu);
5704
5705
5706 dev_len = mtu + dev->hard_header_len;
5707
5708
5709 if (*mtu_len)
5710 xdp_len = *mtu_len + dev->hard_header_len;
5711
5712 xdp_len += len_diff;
5713 if (xdp_len > dev_len)
5714 ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
5715
5716
5717 *mtu_len = mtu;
5718
5719 return ret;
5720}
5721
5722static const struct bpf_func_proto bpf_skb_check_mtu_proto = {
5723 .func = bpf_skb_check_mtu,
5724 .gpl_only = true,
5725 .ret_type = RET_INTEGER,
5726 .arg1_type = ARG_PTR_TO_CTX,
5727 .arg2_type = ARG_ANYTHING,
5728 .arg3_type = ARG_PTR_TO_INT,
5729 .arg4_type = ARG_ANYTHING,
5730 .arg5_type = ARG_ANYTHING,
5731};
5732
5733static const struct bpf_func_proto bpf_xdp_check_mtu_proto = {
5734 .func = bpf_xdp_check_mtu,
5735 .gpl_only = true,
5736 .ret_type = RET_INTEGER,
5737 .arg1_type = ARG_PTR_TO_CTX,
5738 .arg2_type = ARG_ANYTHING,
5739 .arg3_type = ARG_PTR_TO_INT,
5740 .arg4_type = ARG_ANYTHING,
5741 .arg5_type = ARG_ANYTHING,
5742};
5743
5744#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
5745static int bpf_push_seg6_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len)
5746{
5747 int err;
5748 struct ipv6_sr_hdr *srh = (struct ipv6_sr_hdr *)hdr;
5749
5750 if (!seg6_validate_srh(srh, len, false))
5751 return -EINVAL;
5752
5753 switch (type) {
5754 case BPF_LWT_ENCAP_SEG6_INLINE:
5755 if (skb->protocol != htons(ETH_P_IPV6))
5756 return -EBADMSG;
5757
5758 err = seg6_do_srh_inline(skb, srh);
5759 break;
5760 case BPF_LWT_ENCAP_SEG6:
5761 skb_reset_inner_headers(skb);
5762 skb->encapsulation = 1;
5763 err = seg6_do_srh_encap(skb, srh, IPPROTO_IPV6);
5764 break;
5765 default:
5766 return -EINVAL;
5767 }
5768
5769 bpf_compute_data_pointers(skb);
5770 if (err)
5771 return err;
5772
5773 ipv6_hdr(skb)->payload_len = htons(skb->len - sizeof(struct ipv6hdr));
5774 skb_set_transport_header(skb, sizeof(struct ipv6hdr));
5775
5776 return seg6_lookup_nexthop(skb, NULL, 0);
5777}
5778#endif
5779
5780#if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
5781static int bpf_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len,
5782 bool ingress)
5783{
5784 return bpf_lwt_push_ip_encap(skb, hdr, len, ingress);
5785}
5786#endif
5787
5788BPF_CALL_4(bpf_lwt_in_push_encap, struct sk_buff *, skb, u32, type, void *, hdr,
5789 u32, len)
5790{
5791 switch (type) {
5792#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
5793 case BPF_LWT_ENCAP_SEG6:
5794 case BPF_LWT_ENCAP_SEG6_INLINE:
5795 return bpf_push_seg6_encap(skb, type, hdr, len);
5796#endif
5797#if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
5798 case BPF_LWT_ENCAP_IP:
5799 return bpf_push_ip_encap(skb, hdr, len, true );
5800#endif
5801 default:
5802 return -EINVAL;
5803 }
5804}
5805
5806BPF_CALL_4(bpf_lwt_xmit_push_encap, struct sk_buff *, skb, u32, type,
5807 void *, hdr, u32, len)
5808{
5809 switch (type) {
5810#if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
5811 case BPF_LWT_ENCAP_IP:
5812 return bpf_push_ip_encap(skb, hdr, len, false );
5813#endif
5814 default:
5815 return -EINVAL;
5816 }
5817}
5818
5819static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = {
5820 .func = bpf_lwt_in_push_encap,
5821 .gpl_only = false,
5822 .ret_type = RET_INTEGER,
5823 .arg1_type = ARG_PTR_TO_CTX,
5824 .arg2_type = ARG_ANYTHING,
5825 .arg3_type = ARG_PTR_TO_MEM,
5826 .arg4_type = ARG_CONST_SIZE
5827};
5828
5829static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = {
5830 .func = bpf_lwt_xmit_push_encap,
5831 .gpl_only = false,
5832 .ret_type = RET_INTEGER,
5833 .arg1_type = ARG_PTR_TO_CTX,
5834 .arg2_type = ARG_ANYTHING,
5835 .arg3_type = ARG_PTR_TO_MEM,
5836 .arg4_type = ARG_CONST_SIZE
5837};
5838
5839#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
5840BPF_CALL_4(bpf_lwt_seg6_store_bytes, struct sk_buff *, skb, u32, offset,
5841 const void *, from, u32, len)
5842{
5843 struct seg6_bpf_srh_state *srh_state =
5844 this_cpu_ptr(&seg6_bpf_srh_states);
5845 struct ipv6_sr_hdr *srh = srh_state->srh;
5846 void *srh_tlvs, *srh_end, *ptr;
5847 int srhoff = 0;
5848
5849 if (srh == NULL)
5850 return -EINVAL;
5851
5852 srh_tlvs = (void *)((char *)srh + ((srh->first_segment + 1) << 4));
5853 srh_end = (void *)((char *)srh + sizeof(*srh) + srh_state->hdrlen);
5854
5855 ptr = skb->data + offset;
5856 if (ptr >= srh_tlvs && ptr + len <= srh_end)
5857 srh_state->valid = false;
5858 else if (ptr < (void *)&srh->flags ||
5859 ptr + len > (void *)&srh->segments)
5860 return -EFAULT;
5861
5862 if (unlikely(bpf_try_make_writable(skb, offset + len)))
5863 return -EFAULT;
5864 if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0)
5865 return -EINVAL;
5866 srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
5867
5868 memcpy(skb->data + offset, from, len);
5869 return 0;
5870}
5871
5872static const struct bpf_func_proto bpf_lwt_seg6_store_bytes_proto = {
5873 .func = bpf_lwt_seg6_store_bytes,
5874 .gpl_only = false,
5875 .ret_type = RET_INTEGER,
5876 .arg1_type = ARG_PTR_TO_CTX,
5877 .arg2_type = ARG_ANYTHING,
5878 .arg3_type = ARG_PTR_TO_MEM,
5879 .arg4_type = ARG_CONST_SIZE
5880};
5881
5882static void bpf_update_srh_state(struct sk_buff *skb)
5883{
5884 struct seg6_bpf_srh_state *srh_state =
5885 this_cpu_ptr(&seg6_bpf_srh_states);
5886 int srhoff = 0;
5887
5888 if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0) {
5889 srh_state->srh = NULL;
5890 } else {
5891 srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
5892 srh_state->hdrlen = srh_state->srh->hdrlen << 3;
5893 srh_state->valid = true;
5894 }
5895}
5896
5897BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb,
5898 u32, action, void *, param, u32, param_len)
5899{
5900 struct seg6_bpf_srh_state *srh_state =
5901 this_cpu_ptr(&seg6_bpf_srh_states);
5902 int hdroff = 0;
5903 int err;
5904
5905 switch (action) {
5906 case SEG6_LOCAL_ACTION_END_X:
5907 if (!seg6_bpf_has_valid_srh(skb))
5908 return -EBADMSG;
5909 if (param_len != sizeof(struct in6_addr))
5910 return -EINVAL;
5911 return seg6_lookup_nexthop(skb, (struct in6_addr *)param, 0);
5912 case SEG6_LOCAL_ACTION_END_T:
5913 if (!seg6_bpf_has_valid_srh(skb))
5914 return -EBADMSG;
5915 if (param_len != sizeof(int))
5916 return -EINVAL;
5917 return seg6_lookup_nexthop(skb, NULL, *(int *)param);
5918 case SEG6_LOCAL_ACTION_END_DT6:
5919 if (!seg6_bpf_has_valid_srh(skb))
5920 return -EBADMSG;
5921 if (param_len != sizeof(int))
5922 return -EINVAL;
5923
5924 if (ipv6_find_hdr(skb, &hdroff, IPPROTO_IPV6, NULL, NULL) < 0)
5925 return -EBADMSG;
5926 if (!pskb_pull(skb, hdroff))
5927 return -EBADMSG;
5928
5929 skb_postpull_rcsum(skb, skb_network_header(skb), hdroff);
5930 skb_reset_network_header(skb);
5931 skb_reset_transport_header(skb);
5932 skb->encapsulation = 0;
5933
5934 bpf_compute_data_pointers(skb);
5935 bpf_update_srh_state(skb);
5936 return seg6_lookup_nexthop(skb, NULL, *(int *)param);
5937 case SEG6_LOCAL_ACTION_END_B6:
5938 if (srh_state->srh && !seg6_bpf_has_valid_srh(skb))
5939 return -EBADMSG;
5940 err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6_INLINE,
5941 param, param_len);
5942 if (!err)
5943 bpf_update_srh_state(skb);
5944
5945 return err;
5946 case SEG6_LOCAL_ACTION_END_B6_ENCAP:
5947 if (srh_state->srh && !seg6_bpf_has_valid_srh(skb))
5948 return -EBADMSG;
5949 err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6,
5950 param, param_len);
5951 if (!err)
5952 bpf_update_srh_state(skb);
5953
5954 return err;
5955 default:
5956 return -EINVAL;
5957 }
5958}
5959
5960static const struct bpf_func_proto bpf_lwt_seg6_action_proto = {
5961 .func = bpf_lwt_seg6_action,
5962 .gpl_only = false,
5963 .ret_type = RET_INTEGER,
5964 .arg1_type = ARG_PTR_TO_CTX,
5965 .arg2_type = ARG_ANYTHING,
5966 .arg3_type = ARG_PTR_TO_MEM,
5967 .arg4_type = ARG_CONST_SIZE
5968};
5969
5970BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset,
5971 s32, len)
5972{
5973 struct seg6_bpf_srh_state *srh_state =
5974 this_cpu_ptr(&seg6_bpf_srh_states);
5975 struct ipv6_sr_hdr *srh = srh_state->srh;
5976 void *srh_end, *srh_tlvs, *ptr;
5977 struct ipv6hdr *hdr;
5978 int srhoff = 0;
5979 int ret;
5980
5981 if (unlikely(srh == NULL))
5982 return -EINVAL;
5983
5984 srh_tlvs = (void *)((unsigned char *)srh + sizeof(*srh) +
5985 ((srh->first_segment + 1) << 4));
5986 srh_end = (void *)((unsigned char *)srh + sizeof(*srh) +
5987 srh_state->hdrlen);
5988 ptr = skb->data + offset;
5989
5990 if (unlikely(ptr < srh_tlvs || ptr > srh_end))
5991 return -EFAULT;
5992 if (unlikely(len < 0 && (void *)((char *)ptr - len) > srh_end))
5993 return -EFAULT;
5994
5995 if (len > 0) {
5996 ret = skb_cow_head(skb, len);
5997 if (unlikely(ret < 0))
5998 return ret;
5999
6000 ret = bpf_skb_net_hdr_push(skb, offset, len);
6001 } else {
6002 ret = bpf_skb_net_hdr_pop(skb, offset, -1 * len);
6003 }
6004
6005 bpf_compute_data_pointers(skb);
6006 if (unlikely(ret < 0))
6007 return ret;
6008
6009 hdr = (struct ipv6hdr *)skb->data;
6010 hdr->payload_len = htons(skb->len - sizeof(struct ipv6hdr));
6011
6012 if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0)
6013 return -EINVAL;
6014 srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
6015 srh_state->hdrlen += len;
6016 srh_state->valid = false;
6017 return 0;
6018}
6019
6020static const struct bpf_func_proto bpf_lwt_seg6_adjust_srh_proto = {
6021 .func = bpf_lwt_seg6_adjust_srh,
6022 .gpl_only = false,
6023 .ret_type = RET_INTEGER,
6024 .arg1_type = ARG_PTR_TO_CTX,
6025 .arg2_type = ARG_ANYTHING,
6026 .arg3_type = ARG_ANYTHING,
6027};
6028#endif
6029
6030#ifdef CONFIG_INET
6031static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
6032 int dif, int sdif, u8 family, u8 proto)
6033{
6034 bool refcounted = false;
6035 struct sock *sk = NULL;
6036
6037 if (family == AF_INET) {
6038 __be32 src4 = tuple->ipv4.saddr;
6039 __be32 dst4 = tuple->ipv4.daddr;
6040
6041 if (proto == IPPROTO_TCP)
6042 sk = __inet_lookup(net, &tcp_hashinfo, NULL, 0,
6043 src4, tuple->ipv4.sport,
6044 dst4, tuple->ipv4.dport,
6045 dif, sdif, &refcounted);
6046 else
6047 sk = __udp4_lib_lookup(net, src4, tuple->ipv4.sport,
6048 dst4, tuple->ipv4.dport,
6049 dif, sdif, &udp_table, NULL);
6050#if IS_ENABLED(CONFIG_IPV6)
6051 } else {
6052 struct in6_addr *src6 = (struct in6_addr *)&tuple->ipv6.saddr;
6053 struct in6_addr *dst6 = (struct in6_addr *)&tuple->ipv6.daddr;
6054
6055 if (proto == IPPROTO_TCP)
6056 sk = __inet6_lookup(net, &tcp_hashinfo, NULL, 0,
6057 src6, tuple->ipv6.sport,
6058 dst6, ntohs(tuple->ipv6.dport),
6059 dif, sdif, &refcounted);
6060 else if (likely(ipv6_bpf_stub))
6061 sk = ipv6_bpf_stub->udp6_lib_lookup(net,
6062 src6, tuple->ipv6.sport,
6063 dst6, tuple->ipv6.dport,
6064 dif, sdif,
6065 &udp_table, NULL);
6066#endif
6067 }
6068
6069 if (unlikely(sk && !refcounted && !sock_flag(sk, SOCK_RCU_FREE))) {
6070 WARN_ONCE(1, "Found non-RCU, unreferenced socket!");
6071 sk = NULL;
6072 }
6073 return sk;
6074}
6075
6076
6077
6078
6079
6080
6081static struct sock *
6082__bpf_skc_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
6083 struct net *caller_net, u32 ifindex, u8 proto, u64 netns_id,
6084 u64 flags)
6085{
6086 struct sock *sk = NULL;
6087 u8 family = AF_UNSPEC;
6088 struct net *net;
6089 int sdif;
6090
6091 if (len == sizeof(tuple->ipv4))
6092 family = AF_INET;
6093 else if (len == sizeof(tuple->ipv6))
6094 family = AF_INET6;
6095 else
6096 return NULL;
6097
6098 if (unlikely(family == AF_UNSPEC || flags ||
6099 !((s32)netns_id < 0 || netns_id <= S32_MAX)))
6100 goto out;
6101
6102 if (family == AF_INET)
6103 sdif = inet_sdif(skb);
6104 else
6105 sdif = inet6_sdif(skb);
6106
6107 if ((s32)netns_id < 0) {
6108 net = caller_net;
6109 sk = sk_lookup(net, tuple, ifindex, sdif, family, proto);
6110 } else {
6111 net = get_net_ns_by_id(caller_net, netns_id);
6112 if (unlikely(!net))
6113 goto out;
6114 sk = sk_lookup(net, tuple, ifindex, sdif, family, proto);
6115 put_net(net);
6116 }
6117
6118out:
6119 return sk;
6120}
6121
6122static struct sock *
6123__bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
6124 struct net *caller_net, u32 ifindex, u8 proto, u64 netns_id,
6125 u64 flags)
6126{
6127 struct sock *sk = __bpf_skc_lookup(skb, tuple, len, caller_net,
6128 ifindex, proto, netns_id, flags);
6129
6130 if (sk) {
6131 sk = sk_to_full_sk(sk);
6132 if (!sk_fullsock(sk)) {
6133 sock_gen_put(sk);
6134 return NULL;
6135 }
6136 }
6137
6138 return sk;
6139}
6140
6141static struct sock *
6142bpf_skc_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
6143 u8 proto, u64 netns_id, u64 flags)
6144{
6145 struct net *caller_net;
6146 int ifindex;
6147
6148 if (skb->dev) {
6149 caller_net = dev_net(skb->dev);
6150 ifindex = skb->dev->ifindex;
6151 } else {
6152 caller_net = sock_net(skb->sk);
6153 ifindex = 0;
6154 }
6155
6156 return __bpf_skc_lookup(skb, tuple, len, caller_net, ifindex, proto,
6157 netns_id, flags);
6158}
6159
6160static struct sock *
6161bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
6162 u8 proto, u64 netns_id, u64 flags)
6163{
6164 struct sock *sk = bpf_skc_lookup(skb, tuple, len, proto, netns_id,
6165 flags);
6166
6167 if (sk) {
6168 sk = sk_to_full_sk(sk);
6169 if (!sk_fullsock(sk)) {
6170 sock_gen_put(sk);
6171 return NULL;
6172 }
6173 }
6174
6175 return sk;
6176}
6177
6178BPF_CALL_5(bpf_skc_lookup_tcp, struct sk_buff *, skb,
6179 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
6180{
6181 return (unsigned long)bpf_skc_lookup(skb, tuple, len, IPPROTO_TCP,
6182 netns_id, flags);
6183}
6184
6185static const struct bpf_func_proto bpf_skc_lookup_tcp_proto = {
6186 .func = bpf_skc_lookup_tcp,
6187 .gpl_only = false,
6188 .pkt_access = true,
6189 .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
6190 .arg1_type = ARG_PTR_TO_CTX,
6191 .arg2_type = ARG_PTR_TO_MEM,
6192 .arg3_type = ARG_CONST_SIZE,
6193 .arg4_type = ARG_ANYTHING,
6194 .arg5_type = ARG_ANYTHING,
6195};
6196
6197BPF_CALL_5(bpf_sk_lookup_tcp, struct sk_buff *, skb,
6198 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
6199{
6200 return (unsigned long)bpf_sk_lookup(skb, tuple, len, IPPROTO_TCP,
6201 netns_id, flags);
6202}
6203
6204static const struct bpf_func_proto bpf_sk_lookup_tcp_proto = {
6205 .func = bpf_sk_lookup_tcp,
6206 .gpl_only = false,
6207 .pkt_access = true,
6208 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6209 .arg1_type = ARG_PTR_TO_CTX,
6210 .arg2_type = ARG_PTR_TO_MEM,
6211 .arg3_type = ARG_CONST_SIZE,
6212 .arg4_type = ARG_ANYTHING,
6213 .arg5_type = ARG_ANYTHING,
6214};
6215
6216BPF_CALL_5(bpf_sk_lookup_udp, struct sk_buff *, skb,
6217 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
6218{
6219 return (unsigned long)bpf_sk_lookup(skb, tuple, len, IPPROTO_UDP,
6220 netns_id, flags);
6221}
6222
6223static const struct bpf_func_proto bpf_sk_lookup_udp_proto = {
6224 .func = bpf_sk_lookup_udp,
6225 .gpl_only = false,
6226 .pkt_access = true,
6227 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6228 .arg1_type = ARG_PTR_TO_CTX,
6229 .arg2_type = ARG_PTR_TO_MEM,
6230 .arg3_type = ARG_CONST_SIZE,
6231 .arg4_type = ARG_ANYTHING,
6232 .arg5_type = ARG_ANYTHING,
6233};
6234
6235BPF_CALL_1(bpf_sk_release, struct sock *, sk)
6236{
6237 if (sk && sk_is_refcounted(sk))
6238 sock_gen_put(sk);
6239 return 0;
6240}
6241
6242static const struct bpf_func_proto bpf_sk_release_proto = {
6243 .func = bpf_sk_release,
6244 .gpl_only = false,
6245 .ret_type = RET_INTEGER,
6246 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
6247};
6248
6249BPF_CALL_5(bpf_xdp_sk_lookup_udp, struct xdp_buff *, ctx,
6250 struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
6251{
6252 struct net *caller_net = dev_net(ctx->rxq->dev);
6253 int ifindex = ctx->rxq->dev->ifindex;
6254
6255 return (unsigned long)__bpf_sk_lookup(NULL, tuple, len, caller_net,
6256 ifindex, IPPROTO_UDP, netns_id,
6257 flags);
6258}
6259
6260static const struct bpf_func_proto bpf_xdp_sk_lookup_udp_proto = {
6261 .func = bpf_xdp_sk_lookup_udp,
6262 .gpl_only = false,
6263 .pkt_access = true,
6264 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6265 .arg1_type = ARG_PTR_TO_CTX,
6266 .arg2_type = ARG_PTR_TO_MEM,
6267 .arg3_type = ARG_CONST_SIZE,
6268 .arg4_type = ARG_ANYTHING,
6269 .arg5_type = ARG_ANYTHING,
6270};
6271
6272BPF_CALL_5(bpf_xdp_skc_lookup_tcp, struct xdp_buff *, ctx,
6273 struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
6274{
6275 struct net *caller_net = dev_net(ctx->rxq->dev);
6276 int ifindex = ctx->rxq->dev->ifindex;
6277
6278 return (unsigned long)__bpf_skc_lookup(NULL, tuple, len, caller_net,
6279 ifindex, IPPROTO_TCP, netns_id,
6280 flags);
6281}
6282
6283static const struct bpf_func_proto bpf_xdp_skc_lookup_tcp_proto = {
6284 .func = bpf_xdp_skc_lookup_tcp,
6285 .gpl_only = false,
6286 .pkt_access = true,
6287 .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
6288 .arg1_type = ARG_PTR_TO_CTX,
6289 .arg2_type = ARG_PTR_TO_MEM,
6290 .arg3_type = ARG_CONST_SIZE,
6291 .arg4_type = ARG_ANYTHING,
6292 .arg5_type = ARG_ANYTHING,
6293};
6294
6295BPF_CALL_5(bpf_xdp_sk_lookup_tcp, struct xdp_buff *, ctx,
6296 struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
6297{
6298 struct net *caller_net = dev_net(ctx->rxq->dev);
6299 int ifindex = ctx->rxq->dev->ifindex;
6300
6301 return (unsigned long)__bpf_sk_lookup(NULL, tuple, len, caller_net,
6302 ifindex, IPPROTO_TCP, netns_id,
6303 flags);
6304}
6305
6306static const struct bpf_func_proto bpf_xdp_sk_lookup_tcp_proto = {
6307 .func = bpf_xdp_sk_lookup_tcp,
6308 .gpl_only = false,
6309 .pkt_access = true,
6310 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6311 .arg1_type = ARG_PTR_TO_CTX,
6312 .arg2_type = ARG_PTR_TO_MEM,
6313 .arg3_type = ARG_CONST_SIZE,
6314 .arg4_type = ARG_ANYTHING,
6315 .arg5_type = ARG_ANYTHING,
6316};
6317
6318BPF_CALL_5(bpf_sock_addr_skc_lookup_tcp, struct bpf_sock_addr_kern *, ctx,
6319 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
6320{
6321 return (unsigned long)__bpf_skc_lookup(NULL, tuple, len,
6322 sock_net(ctx->sk), 0,
6323 IPPROTO_TCP, netns_id, flags);
6324}
6325
6326static const struct bpf_func_proto bpf_sock_addr_skc_lookup_tcp_proto = {
6327 .func = bpf_sock_addr_skc_lookup_tcp,
6328 .gpl_only = false,
6329 .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
6330 .arg1_type = ARG_PTR_TO_CTX,
6331 .arg2_type = ARG_PTR_TO_MEM,
6332 .arg3_type = ARG_CONST_SIZE,
6333 .arg4_type = ARG_ANYTHING,
6334 .arg5_type = ARG_ANYTHING,
6335};
6336
6337BPF_CALL_5(bpf_sock_addr_sk_lookup_tcp, struct bpf_sock_addr_kern *, ctx,
6338 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
6339{
6340 return (unsigned long)__bpf_sk_lookup(NULL, tuple, len,
6341 sock_net(ctx->sk), 0, IPPROTO_TCP,
6342 netns_id, flags);
6343}
6344
6345static const struct bpf_func_proto bpf_sock_addr_sk_lookup_tcp_proto = {
6346 .func = bpf_sock_addr_sk_lookup_tcp,
6347 .gpl_only = false,
6348 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6349 .arg1_type = ARG_PTR_TO_CTX,
6350 .arg2_type = ARG_PTR_TO_MEM,
6351 .arg3_type = ARG_CONST_SIZE,
6352 .arg4_type = ARG_ANYTHING,
6353 .arg5_type = ARG_ANYTHING,
6354};
6355
6356BPF_CALL_5(bpf_sock_addr_sk_lookup_udp, struct bpf_sock_addr_kern *, ctx,
6357 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
6358{
6359 return (unsigned long)__bpf_sk_lookup(NULL, tuple, len,
6360 sock_net(ctx->sk), 0, IPPROTO_UDP,
6361 netns_id, flags);
6362}
6363
6364static const struct bpf_func_proto bpf_sock_addr_sk_lookup_udp_proto = {
6365 .func = bpf_sock_addr_sk_lookup_udp,
6366 .gpl_only = false,
6367 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6368 .arg1_type = ARG_PTR_TO_CTX,
6369 .arg2_type = ARG_PTR_TO_MEM,
6370 .arg3_type = ARG_CONST_SIZE,
6371 .arg4_type = ARG_ANYTHING,
6372 .arg5_type = ARG_ANYTHING,
6373};
6374
6375bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
6376 struct bpf_insn_access_aux *info)
6377{
6378 if (off < 0 || off >= offsetofend(struct bpf_tcp_sock,
6379 icsk_retransmits))
6380 return false;
6381
6382 if (off % size != 0)
6383 return false;
6384
6385 switch (off) {
6386 case offsetof(struct bpf_tcp_sock, bytes_received):
6387 case offsetof(struct bpf_tcp_sock, bytes_acked):
6388 return size == sizeof(__u64);
6389 default:
6390 return size == sizeof(__u32);
6391 }
6392}
6393
6394u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
6395 const struct bpf_insn *si,
6396 struct bpf_insn *insn_buf,
6397 struct bpf_prog *prog, u32 *target_size)
6398{
6399 struct bpf_insn *insn = insn_buf;
6400
6401#define BPF_TCP_SOCK_GET_COMMON(FIELD) \
6402 do { \
6403 BUILD_BUG_ON(sizeof_field(struct tcp_sock, FIELD) > \
6404 sizeof_field(struct bpf_tcp_sock, FIELD)); \
6405 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct tcp_sock, FIELD),\
6406 si->dst_reg, si->src_reg, \
6407 offsetof(struct tcp_sock, FIELD)); \
6408 } while (0)
6409
6410#define BPF_INET_SOCK_GET_COMMON(FIELD) \
6411 do { \
6412 BUILD_BUG_ON(sizeof_field(struct inet_connection_sock, \
6413 FIELD) > \
6414 sizeof_field(struct bpf_tcp_sock, FIELD)); \
6415 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
6416 struct inet_connection_sock, \
6417 FIELD), \
6418 si->dst_reg, si->src_reg, \
6419 offsetof( \
6420 struct inet_connection_sock, \
6421 FIELD)); \
6422 } while (0)
6423
6424 if (insn > insn_buf)
6425 return insn - insn_buf;
6426
6427 switch (si->off) {
6428 case offsetof(struct bpf_tcp_sock, rtt_min):
6429 BUILD_BUG_ON(sizeof_field(struct tcp_sock, rtt_min) !=
6430 sizeof(struct minmax));
6431 BUILD_BUG_ON(sizeof(struct minmax) <
6432 sizeof(struct minmax_sample));
6433
6434 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
6435 offsetof(struct tcp_sock, rtt_min) +
6436 offsetof(struct minmax_sample, v));
6437 break;
6438 case offsetof(struct bpf_tcp_sock, snd_cwnd):
6439 BPF_TCP_SOCK_GET_COMMON(snd_cwnd);
6440 break;
6441 case offsetof(struct bpf_tcp_sock, srtt_us):
6442 BPF_TCP_SOCK_GET_COMMON(srtt_us);
6443 break;
6444 case offsetof(struct bpf_tcp_sock, snd_ssthresh):
6445 BPF_TCP_SOCK_GET_COMMON(snd_ssthresh);
6446 break;
6447 case offsetof(struct bpf_tcp_sock, rcv_nxt):
6448 BPF_TCP_SOCK_GET_COMMON(rcv_nxt);
6449 break;
6450 case offsetof(struct bpf_tcp_sock, snd_nxt):
6451 BPF_TCP_SOCK_GET_COMMON(snd_nxt);
6452 break;
6453 case offsetof(struct bpf_tcp_sock, snd_una):
6454 BPF_TCP_SOCK_GET_COMMON(snd_una);
6455 break;
6456 case offsetof(struct bpf_tcp_sock, mss_cache):
6457 BPF_TCP_SOCK_GET_COMMON(mss_cache);
6458 break;
6459 case offsetof(struct bpf_tcp_sock, ecn_flags):
6460 BPF_TCP_SOCK_GET_COMMON(ecn_flags);
6461 break;
6462 case offsetof(struct bpf_tcp_sock, rate_delivered):
6463 BPF_TCP_SOCK_GET_COMMON(rate_delivered);
6464 break;
6465 case offsetof(struct bpf_tcp_sock, rate_interval_us):
6466 BPF_TCP_SOCK_GET_COMMON(rate_interval_us);
6467 break;
6468 case offsetof(struct bpf_tcp_sock, packets_out):
6469 BPF_TCP_SOCK_GET_COMMON(packets_out);
6470 break;
6471 case offsetof(struct bpf_tcp_sock, retrans_out):
6472 BPF_TCP_SOCK_GET_COMMON(retrans_out);
6473 break;
6474 case offsetof(struct bpf_tcp_sock, total_retrans):
6475 BPF_TCP_SOCK_GET_COMMON(total_retrans);
6476 break;
6477 case offsetof(struct bpf_tcp_sock, segs_in):
6478 BPF_TCP_SOCK_GET_COMMON(segs_in);
6479 break;
6480 case offsetof(struct bpf_tcp_sock, data_segs_in):
6481 BPF_TCP_SOCK_GET_COMMON(data_segs_in);
6482 break;
6483 case offsetof(struct bpf_tcp_sock, segs_out):
6484 BPF_TCP_SOCK_GET_COMMON(segs_out);
6485 break;
6486 case offsetof(struct bpf_tcp_sock, data_segs_out):
6487 BPF_TCP_SOCK_GET_COMMON(data_segs_out);
6488 break;
6489 case offsetof(struct bpf_tcp_sock, lost_out):
6490 BPF_TCP_SOCK_GET_COMMON(lost_out);
6491 break;
6492 case offsetof(struct bpf_tcp_sock, sacked_out):
6493 BPF_TCP_SOCK_GET_COMMON(sacked_out);
6494 break;
6495 case offsetof(struct bpf_tcp_sock, bytes_received):
6496 BPF_TCP_SOCK_GET_COMMON(bytes_received);
6497 break;
6498 case offsetof(struct bpf_tcp_sock, bytes_acked):
6499 BPF_TCP_SOCK_GET_COMMON(bytes_acked);
6500 break;
6501 case offsetof(struct bpf_tcp_sock, dsack_dups):
6502 BPF_TCP_SOCK_GET_COMMON(dsack_dups);
6503 break;
6504 case offsetof(struct bpf_tcp_sock, delivered):
6505 BPF_TCP_SOCK_GET_COMMON(delivered);
6506 break;
6507 case offsetof(struct bpf_tcp_sock, delivered_ce):
6508 BPF_TCP_SOCK_GET_COMMON(delivered_ce);
6509 break;
6510 case offsetof(struct bpf_tcp_sock, icsk_retransmits):
6511 BPF_INET_SOCK_GET_COMMON(icsk_retransmits);
6512 break;
6513 }
6514
6515 return insn - insn_buf;
6516}
6517
6518BPF_CALL_1(bpf_tcp_sock, struct sock *, sk)
6519{
6520 if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
6521 return (unsigned long)sk;
6522
6523 return (unsigned long)NULL;
6524}
6525
6526const struct bpf_func_proto bpf_tcp_sock_proto = {
6527 .func = bpf_tcp_sock,
6528 .gpl_only = false,
6529 .ret_type = RET_PTR_TO_TCP_SOCK_OR_NULL,
6530 .arg1_type = ARG_PTR_TO_SOCK_COMMON,
6531};
6532
6533BPF_CALL_1(bpf_get_listener_sock, struct sock *, sk)
6534{
6535 sk = sk_to_full_sk(sk);
6536
6537 if (sk->sk_state == TCP_LISTEN && sock_flag(sk, SOCK_RCU_FREE))
6538 return (unsigned long)sk;
6539
6540 return (unsigned long)NULL;
6541}
6542
6543static const struct bpf_func_proto bpf_get_listener_sock_proto = {
6544 .func = bpf_get_listener_sock,
6545 .gpl_only = false,
6546 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6547 .arg1_type = ARG_PTR_TO_SOCK_COMMON,
6548};
6549
6550BPF_CALL_1(bpf_skb_ecn_set_ce, struct sk_buff *, skb)
6551{
6552 unsigned int iphdr_len;
6553
6554 switch (skb_protocol(skb, true)) {
6555 case cpu_to_be16(ETH_P_IP):
6556 iphdr_len = sizeof(struct iphdr);
6557 break;
6558 case cpu_to_be16(ETH_P_IPV6):
6559 iphdr_len = sizeof(struct ipv6hdr);
6560 break;
6561 default:
6562 return 0;
6563 }
6564
6565 if (skb_headlen(skb) < iphdr_len)
6566 return 0;
6567
6568 if (skb_cloned(skb) && !skb_clone_writable(skb, iphdr_len))
6569 return 0;
6570
6571 return INET_ECN_set_ce(skb);
6572}
6573
6574bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
6575 struct bpf_insn_access_aux *info)
6576{
6577 if (off < 0 || off >= offsetofend(struct bpf_xdp_sock, queue_id))
6578 return false;
6579
6580 if (off % size != 0)
6581 return false;
6582
6583 switch (off) {
6584 default:
6585 return size == sizeof(__u32);
6586 }
6587}
6588
6589u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
6590 const struct bpf_insn *si,
6591 struct bpf_insn *insn_buf,
6592 struct bpf_prog *prog, u32 *target_size)
6593{
6594 struct bpf_insn *insn = insn_buf;
6595
6596#define BPF_XDP_SOCK_GET(FIELD) \
6597 do { \
6598 BUILD_BUG_ON(sizeof_field(struct xdp_sock, FIELD) > \
6599 sizeof_field(struct bpf_xdp_sock, FIELD)); \
6600 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_sock, FIELD),\
6601 si->dst_reg, si->src_reg, \
6602 offsetof(struct xdp_sock, FIELD)); \
6603 } while (0)
6604
6605 switch (si->off) {
6606 case offsetof(struct bpf_xdp_sock, queue_id):
6607 BPF_XDP_SOCK_GET(queue_id);
6608 break;
6609 }
6610
6611 return insn - insn_buf;
6612}
6613
6614static const struct bpf_func_proto bpf_skb_ecn_set_ce_proto = {
6615 .func = bpf_skb_ecn_set_ce,
6616 .gpl_only = false,
6617 .ret_type = RET_INTEGER,
6618 .arg1_type = ARG_PTR_TO_CTX,
6619};
6620
6621BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len,
6622 struct tcphdr *, th, u32, th_len)
6623{
6624#ifdef CONFIG_SYN_COOKIES
6625 u32 cookie;
6626 int ret;
6627
6628 if (unlikely(!sk || th_len < sizeof(*th)))
6629 return -EINVAL;
6630
6631
6632 if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN)
6633 return -EINVAL;
6634
6635 if (!sock_net(sk)->ipv4.sysctl_tcp_syncookies)
6636 return -EINVAL;
6637
6638 if (!th->ack || th->rst || th->syn)
6639 return -ENOENT;
6640
6641 if (tcp_synq_no_recent_overflow(sk))
6642 return -ENOENT;
6643
6644 cookie = ntohl(th->ack_seq) - 1;
6645
6646 switch (sk->sk_family) {
6647 case AF_INET:
6648 if (unlikely(iph_len < sizeof(struct iphdr)))
6649 return -EINVAL;
6650
6651 ret = __cookie_v4_check((struct iphdr *)iph, th, cookie);
6652 break;
6653
6654#if IS_BUILTIN(CONFIG_IPV6)
6655 case AF_INET6:
6656 if (unlikely(iph_len < sizeof(struct ipv6hdr)))
6657 return -EINVAL;
6658
6659 ret = __cookie_v6_check((struct ipv6hdr *)iph, th, cookie);
6660 break;
6661#endif
6662
6663 default:
6664 return -EPROTONOSUPPORT;
6665 }
6666
6667 if (ret > 0)
6668 return 0;
6669
6670 return -ENOENT;
6671#else
6672 return -ENOTSUPP;
6673#endif
6674}
6675
6676static const struct bpf_func_proto bpf_tcp_check_syncookie_proto = {
6677 .func = bpf_tcp_check_syncookie,
6678 .gpl_only = true,
6679 .pkt_access = true,
6680 .ret_type = RET_INTEGER,
6681 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
6682 .arg2_type = ARG_PTR_TO_MEM,
6683 .arg3_type = ARG_CONST_SIZE,
6684 .arg4_type = ARG_PTR_TO_MEM,
6685 .arg5_type = ARG_CONST_SIZE,
6686};
6687
6688BPF_CALL_5(bpf_tcp_gen_syncookie, struct sock *, sk, void *, iph, u32, iph_len,
6689 struct tcphdr *, th, u32, th_len)
6690{
6691#ifdef CONFIG_SYN_COOKIES
6692 u32 cookie;
6693 u16 mss;
6694
6695 if (unlikely(!sk || th_len < sizeof(*th) || th_len != th->doff * 4))
6696 return -EINVAL;
6697
6698 if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN)
6699 return -EINVAL;
6700
6701 if (!sock_net(sk)->ipv4.sysctl_tcp_syncookies)
6702 return -ENOENT;
6703
6704 if (!th->syn || th->ack || th->fin || th->rst)
6705 return -EINVAL;
6706
6707 if (unlikely(iph_len < sizeof(struct iphdr)))
6708 return -EINVAL;
6709
6710
6711
6712
6713 switch (((struct iphdr *)iph)->version) {
6714 case 4:
6715 if (sk->sk_family == AF_INET6 && sk->sk_ipv6only)
6716 return -EINVAL;
6717
6718 mss = tcp_v4_get_syncookie(sk, iph, th, &cookie);
6719 break;
6720
6721#if IS_BUILTIN(CONFIG_IPV6)
6722 case 6:
6723 if (unlikely(iph_len < sizeof(struct ipv6hdr)))
6724 return -EINVAL;
6725
6726 if (sk->sk_family != AF_INET6)
6727 return -EINVAL;
6728
6729 mss = tcp_v6_get_syncookie(sk, iph, th, &cookie);
6730 break;
6731#endif
6732
6733 default:
6734 return -EPROTONOSUPPORT;
6735 }
6736 if (mss == 0)
6737 return -ENOENT;
6738
6739 return cookie | ((u64)mss << 32);
6740#else
6741 return -EOPNOTSUPP;
6742#endif
6743}
6744
6745static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = {
6746 .func = bpf_tcp_gen_syncookie,
6747 .gpl_only = true,
6748 .pkt_access = true,
6749 .ret_type = RET_INTEGER,
6750 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
6751 .arg2_type = ARG_PTR_TO_MEM,
6752 .arg3_type = ARG_CONST_SIZE,
6753 .arg4_type = ARG_PTR_TO_MEM,
6754 .arg5_type = ARG_CONST_SIZE,
6755};
6756
6757BPF_CALL_3(bpf_sk_assign, struct sk_buff *, skb, struct sock *, sk, u64, flags)
6758{
6759 if (!sk || flags != 0)
6760 return -EINVAL;
6761 if (!skb_at_tc_ingress(skb))
6762 return -EOPNOTSUPP;
6763 if (unlikely(dev_net(skb->dev) != sock_net(sk)))
6764 return -ENETUNREACH;
6765 if (unlikely(sk_fullsock(sk) && sk->sk_reuseport))
6766 return -ESOCKTNOSUPPORT;
6767 if (sk_is_refcounted(sk) &&
6768 unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
6769 return -ENOENT;
6770
6771 skb_orphan(skb);
6772 skb->sk = sk;
6773 skb->destructor = sock_pfree;
6774
6775 return 0;
6776}
6777
6778static const struct bpf_func_proto bpf_sk_assign_proto = {
6779 .func = bpf_sk_assign,
6780 .gpl_only = false,
6781 .ret_type = RET_INTEGER,
6782 .arg1_type = ARG_PTR_TO_CTX,
6783 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
6784 .arg3_type = ARG_ANYTHING,
6785};
6786
6787static const u8 *bpf_search_tcp_opt(const u8 *op, const u8 *opend,
6788 u8 search_kind, const u8 *magic,
6789 u8 magic_len, bool *eol)
6790{
6791 u8 kind, kind_len;
6792
6793 *eol = false;
6794
6795 while (op < opend) {
6796 kind = op[0];
6797
6798 if (kind == TCPOPT_EOL) {
6799 *eol = true;
6800 return ERR_PTR(-ENOMSG);
6801 } else if (kind == TCPOPT_NOP) {
6802 op++;
6803 continue;
6804 }
6805
6806 if (opend - op < 2 || opend - op < op[1] || op[1] < 2)
6807
6808
6809
6810
6811 return ERR_PTR(-EFAULT);
6812
6813 kind_len = op[1];
6814 if (search_kind == kind) {
6815 if (!magic_len)
6816 return op;
6817
6818 if (magic_len > kind_len - 2)
6819 return ERR_PTR(-ENOMSG);
6820
6821 if (!memcmp(&op[2], magic, magic_len))
6822 return op;
6823 }
6824
6825 op += kind_len;
6826 }
6827
6828 return ERR_PTR(-ENOMSG);
6829}
6830
6831BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
6832 void *, search_res, u32, len, u64, flags)
6833{
6834 bool eol, load_syn = flags & BPF_LOAD_HDR_OPT_TCP_SYN;
6835 const u8 *op, *opend, *magic, *search = search_res;
6836 u8 search_kind, search_len, copy_len, magic_len;
6837 int ret;
6838
6839
6840
6841
6842
6843 if (len < 2 || flags & ~BPF_LOAD_HDR_OPT_TCP_SYN)
6844 return -EINVAL;
6845
6846 search_kind = search[0];
6847 search_len = search[1];
6848
6849 if (search_len > len || search_kind == TCPOPT_NOP ||
6850 search_kind == TCPOPT_EOL)
6851 return -EINVAL;
6852
6853 if (search_kind == TCPOPT_EXP || search_kind == 253) {
6854
6855 if (search_len != 4 && search_len != 6)
6856 return -EINVAL;
6857 magic = &search[2];
6858 magic_len = search_len - 2;
6859 } else {
6860 if (search_len)
6861 return -EINVAL;
6862 magic = NULL;
6863 magic_len = 0;
6864 }
6865
6866 if (load_syn) {
6867 ret = bpf_sock_ops_get_syn(bpf_sock, TCP_BPF_SYN, &op);
6868 if (ret < 0)
6869 return ret;
6870
6871 opend = op + ret;
6872 op += sizeof(struct tcphdr);
6873 } else {
6874 if (!bpf_sock->skb ||
6875 bpf_sock->op == BPF_SOCK_OPS_HDR_OPT_LEN_CB)
6876
6877 return -EPERM;
6878
6879 opend = bpf_sock->skb_data_end;
6880 op = bpf_sock->skb->data + sizeof(struct tcphdr);
6881 }
6882
6883 op = bpf_search_tcp_opt(op, opend, search_kind, magic, magic_len,
6884 &eol);
6885 if (IS_ERR(op))
6886 return PTR_ERR(op);
6887
6888 copy_len = op[1];
6889 ret = copy_len;
6890 if (copy_len > len) {
6891 ret = -ENOSPC;
6892 copy_len = len;
6893 }
6894
6895 memcpy(search_res, op, copy_len);
6896 return ret;
6897}
6898
6899static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = {
6900 .func = bpf_sock_ops_load_hdr_opt,
6901 .gpl_only = false,
6902 .ret_type = RET_INTEGER,
6903 .arg1_type = ARG_PTR_TO_CTX,
6904 .arg2_type = ARG_PTR_TO_MEM,
6905 .arg3_type = ARG_CONST_SIZE,
6906 .arg4_type = ARG_ANYTHING,
6907};
6908
6909BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
6910 const void *, from, u32, len, u64, flags)
6911{
6912 u8 new_kind, new_kind_len, magic_len = 0, *opend;
6913 const u8 *op, *new_op, *magic = NULL;
6914 struct sk_buff *skb;
6915 bool eol;
6916
6917 if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB)
6918 return -EPERM;
6919
6920 if (len < 2 || flags)
6921 return -EINVAL;
6922
6923 new_op = from;
6924 new_kind = new_op[0];
6925 new_kind_len = new_op[1];
6926
6927 if (new_kind_len > len || new_kind == TCPOPT_NOP ||
6928 new_kind == TCPOPT_EOL)
6929 return -EINVAL;
6930
6931 if (new_kind_len > bpf_sock->remaining_opt_len)
6932 return -ENOSPC;
6933
6934
6935 if (new_kind == TCPOPT_EXP || new_kind == 253) {
6936 if (new_kind_len < 4)
6937 return -EINVAL;
6938
6939
6940
6941
6942
6943
6944
6945 magic = &new_op[2];
6946 magic_len = 2;
6947 }
6948
6949
6950 skb = bpf_sock->skb;
6951 op = skb->data + sizeof(struct tcphdr);
6952 opend = bpf_sock->skb_data_end;
6953
6954 op = bpf_search_tcp_opt(op, opend, new_kind, magic, magic_len,
6955 &eol);
6956 if (!IS_ERR(op))
6957 return -EEXIST;
6958
6959 if (PTR_ERR(op) != -ENOMSG)
6960 return PTR_ERR(op);
6961
6962 if (eol)
6963
6964
6965
6966 return -ENOSPC;
6967
6968
6969 memcpy(opend, from, new_kind_len);
6970
6971 bpf_sock->remaining_opt_len -= new_kind_len;
6972 bpf_sock->skb_data_end += new_kind_len;
6973
6974 return 0;
6975}
6976
6977static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = {
6978 .func = bpf_sock_ops_store_hdr_opt,
6979 .gpl_only = false,
6980 .ret_type = RET_INTEGER,
6981 .arg1_type = ARG_PTR_TO_CTX,
6982 .arg2_type = ARG_PTR_TO_MEM,
6983 .arg3_type = ARG_CONST_SIZE,
6984 .arg4_type = ARG_ANYTHING,
6985};
6986
6987BPF_CALL_3(bpf_sock_ops_reserve_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
6988 u32, len, u64, flags)
6989{
6990 if (bpf_sock->op != BPF_SOCK_OPS_HDR_OPT_LEN_CB)
6991 return -EPERM;
6992
6993 if (flags || len < 2)
6994 return -EINVAL;
6995
6996 if (len > bpf_sock->remaining_opt_len)
6997 return -ENOSPC;
6998
6999 bpf_sock->remaining_opt_len -= len;
7000
7001 return 0;
7002}
7003
7004static const struct bpf_func_proto bpf_sock_ops_reserve_hdr_opt_proto = {
7005 .func = bpf_sock_ops_reserve_hdr_opt,
7006 .gpl_only = false,
7007 .ret_type = RET_INTEGER,
7008 .arg1_type = ARG_PTR_TO_CTX,
7009 .arg2_type = ARG_ANYTHING,
7010 .arg3_type = ARG_ANYTHING,
7011};
7012
7013#endif
7014
7015bool bpf_helper_changes_pkt_data(void *func)
7016{
7017 if (func == bpf_skb_vlan_push ||
7018 func == bpf_skb_vlan_pop ||
7019 func == bpf_skb_store_bytes ||
7020 func == bpf_skb_change_proto ||
7021 func == bpf_skb_change_head ||
7022 func == sk_skb_change_head ||
7023 func == bpf_skb_change_tail ||
7024 func == sk_skb_change_tail ||
7025 func == bpf_skb_adjust_room ||
7026 func == sk_skb_adjust_room ||
7027 func == bpf_skb_pull_data ||
7028 func == sk_skb_pull_data ||
7029 func == bpf_clone_redirect ||
7030 func == bpf_l3_csum_replace ||
7031 func == bpf_l4_csum_replace ||
7032 func == bpf_xdp_adjust_head ||
7033 func == bpf_xdp_adjust_meta ||
7034 func == bpf_msg_pull_data ||
7035 func == bpf_msg_push_data ||
7036 func == bpf_msg_pop_data ||
7037 func == bpf_xdp_adjust_tail ||
7038#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
7039 func == bpf_lwt_seg6_store_bytes ||
7040 func == bpf_lwt_seg6_adjust_srh ||
7041 func == bpf_lwt_seg6_action ||
7042#endif
7043#ifdef CONFIG_INET
7044 func == bpf_sock_ops_store_hdr_opt ||
7045#endif
7046 func == bpf_lwt_in_push_encap ||
7047 func == bpf_lwt_xmit_push_encap)
7048 return true;
7049
7050 return false;
7051}
7052
7053const struct bpf_func_proto bpf_event_output_data_proto __weak;
7054const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto __weak;
7055
7056static const struct bpf_func_proto *
7057sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7058{
7059 switch (func_id) {
7060
7061
7062
7063 case BPF_FUNC_get_current_uid_gid:
7064 return &bpf_get_current_uid_gid_proto;
7065 case BPF_FUNC_get_local_storage:
7066 return &bpf_get_local_storage_proto;
7067 case BPF_FUNC_get_socket_cookie:
7068 return &bpf_get_socket_cookie_sock_proto;
7069 case BPF_FUNC_get_netns_cookie:
7070 return &bpf_get_netns_cookie_sock_proto;
7071 case BPF_FUNC_perf_event_output:
7072 return &bpf_event_output_data_proto;
7073 case BPF_FUNC_get_current_pid_tgid:
7074 return &bpf_get_current_pid_tgid_proto;
7075 case BPF_FUNC_get_current_comm:
7076 return &bpf_get_current_comm_proto;
7077#ifdef CONFIG_CGROUPS
7078 case BPF_FUNC_get_current_cgroup_id:
7079 return &bpf_get_current_cgroup_id_proto;
7080 case BPF_FUNC_get_current_ancestor_cgroup_id:
7081 return &bpf_get_current_ancestor_cgroup_id_proto;
7082#endif
7083#ifdef CONFIG_CGROUP_NET_CLASSID
7084 case BPF_FUNC_get_cgroup_classid:
7085 return &bpf_get_cgroup_classid_curr_proto;
7086#endif
7087 case BPF_FUNC_sk_storage_get:
7088 return &bpf_sk_storage_get_cg_sock_proto;
7089 default:
7090 return bpf_base_func_proto(func_id);
7091 }
7092}
7093
7094static const struct bpf_func_proto *
7095sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7096{
7097 switch (func_id) {
7098
7099
7100
7101 case BPF_FUNC_get_current_uid_gid:
7102 return &bpf_get_current_uid_gid_proto;
7103 case BPF_FUNC_bind:
7104 switch (prog->expected_attach_type) {
7105 case BPF_CGROUP_INET4_CONNECT:
7106 case BPF_CGROUP_INET6_CONNECT:
7107 return &bpf_bind_proto;
7108 default:
7109 return NULL;
7110 }
7111 case BPF_FUNC_get_socket_cookie:
7112 return &bpf_get_socket_cookie_sock_addr_proto;
7113 case BPF_FUNC_get_netns_cookie:
7114 return &bpf_get_netns_cookie_sock_addr_proto;
7115 case BPF_FUNC_get_local_storage:
7116 return &bpf_get_local_storage_proto;
7117 case BPF_FUNC_perf_event_output:
7118 return &bpf_event_output_data_proto;
7119 case BPF_FUNC_get_current_pid_tgid:
7120 return &bpf_get_current_pid_tgid_proto;
7121 case BPF_FUNC_get_current_comm:
7122 return &bpf_get_current_comm_proto;
7123#ifdef CONFIG_CGROUPS
7124 case BPF_FUNC_get_current_cgroup_id:
7125 return &bpf_get_current_cgroup_id_proto;
7126 case BPF_FUNC_get_current_ancestor_cgroup_id:
7127 return &bpf_get_current_ancestor_cgroup_id_proto;
7128#endif
7129#ifdef CONFIG_CGROUP_NET_CLASSID
7130 case BPF_FUNC_get_cgroup_classid:
7131 return &bpf_get_cgroup_classid_curr_proto;
7132#endif
7133#ifdef CONFIG_INET
7134 case BPF_FUNC_sk_lookup_tcp:
7135 return &bpf_sock_addr_sk_lookup_tcp_proto;
7136 case BPF_FUNC_sk_lookup_udp:
7137 return &bpf_sock_addr_sk_lookup_udp_proto;
7138 case BPF_FUNC_sk_release:
7139 return &bpf_sk_release_proto;
7140 case BPF_FUNC_skc_lookup_tcp:
7141 return &bpf_sock_addr_skc_lookup_tcp_proto;
7142#endif
7143 case BPF_FUNC_sk_storage_get:
7144 return &bpf_sk_storage_get_proto;
7145 case BPF_FUNC_sk_storage_delete:
7146 return &bpf_sk_storage_delete_proto;
7147 case BPF_FUNC_setsockopt:
7148 switch (prog->expected_attach_type) {
7149 case BPF_CGROUP_INET4_BIND:
7150 case BPF_CGROUP_INET6_BIND:
7151 case BPF_CGROUP_INET4_CONNECT:
7152 case BPF_CGROUP_INET6_CONNECT:
7153 case BPF_CGROUP_UDP4_RECVMSG:
7154 case BPF_CGROUP_UDP6_RECVMSG:
7155 case BPF_CGROUP_UDP4_SENDMSG:
7156 case BPF_CGROUP_UDP6_SENDMSG:
7157 case BPF_CGROUP_INET4_GETPEERNAME:
7158 case BPF_CGROUP_INET6_GETPEERNAME:
7159 case BPF_CGROUP_INET4_GETSOCKNAME:
7160 case BPF_CGROUP_INET6_GETSOCKNAME:
7161 return &bpf_sock_addr_setsockopt_proto;
7162 default:
7163 return NULL;
7164 }
7165 case BPF_FUNC_getsockopt:
7166 switch (prog->expected_attach_type) {
7167 case BPF_CGROUP_INET4_BIND:
7168 case BPF_CGROUP_INET6_BIND:
7169 case BPF_CGROUP_INET4_CONNECT:
7170 case BPF_CGROUP_INET6_CONNECT:
7171 case BPF_CGROUP_UDP4_RECVMSG:
7172 case BPF_CGROUP_UDP6_RECVMSG:
7173 case BPF_CGROUP_UDP4_SENDMSG:
7174 case BPF_CGROUP_UDP6_SENDMSG:
7175 case BPF_CGROUP_INET4_GETPEERNAME:
7176 case BPF_CGROUP_INET6_GETPEERNAME:
7177 case BPF_CGROUP_INET4_GETSOCKNAME:
7178 case BPF_CGROUP_INET6_GETSOCKNAME:
7179 return &bpf_sock_addr_getsockopt_proto;
7180 default:
7181 return NULL;
7182 }
7183 default:
7184 return bpf_sk_base_func_proto(func_id);
7185 }
7186}
7187
7188static const struct bpf_func_proto *
7189sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7190{
7191 switch (func_id) {
7192 case BPF_FUNC_skb_load_bytes:
7193 return &bpf_skb_load_bytes_proto;
7194 case BPF_FUNC_skb_load_bytes_relative:
7195 return &bpf_skb_load_bytes_relative_proto;
7196 case BPF_FUNC_get_socket_cookie:
7197 return &bpf_get_socket_cookie_proto;
7198 case BPF_FUNC_get_socket_uid:
7199 return &bpf_get_socket_uid_proto;
7200 case BPF_FUNC_perf_event_output:
7201 return &bpf_skb_event_output_proto;
7202 default:
7203 return bpf_sk_base_func_proto(func_id);
7204 }
7205}
7206
7207const struct bpf_func_proto bpf_sk_storage_get_proto __weak;
7208const struct bpf_func_proto bpf_sk_storage_delete_proto __weak;
7209
7210static const struct bpf_func_proto *
7211cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7212{
7213 switch (func_id) {
7214 case BPF_FUNC_get_local_storage:
7215 return &bpf_get_local_storage_proto;
7216 case BPF_FUNC_sk_fullsock:
7217 return &bpf_sk_fullsock_proto;
7218 case BPF_FUNC_sk_storage_get:
7219 return &bpf_sk_storage_get_proto;
7220 case BPF_FUNC_sk_storage_delete:
7221 return &bpf_sk_storage_delete_proto;
7222 case BPF_FUNC_perf_event_output:
7223 return &bpf_skb_event_output_proto;
7224#ifdef CONFIG_SOCK_CGROUP_DATA
7225 case BPF_FUNC_skb_cgroup_id:
7226 return &bpf_skb_cgroup_id_proto;
7227 case BPF_FUNC_skb_ancestor_cgroup_id:
7228 return &bpf_skb_ancestor_cgroup_id_proto;
7229 case BPF_FUNC_sk_cgroup_id:
7230 return &bpf_sk_cgroup_id_proto;
7231 case BPF_FUNC_sk_ancestor_cgroup_id:
7232 return &bpf_sk_ancestor_cgroup_id_proto;
7233#endif
7234#ifdef CONFIG_INET
7235 case BPF_FUNC_sk_lookup_tcp:
7236 return &bpf_sk_lookup_tcp_proto;
7237 case BPF_FUNC_sk_lookup_udp:
7238 return &bpf_sk_lookup_udp_proto;
7239 case BPF_FUNC_sk_release:
7240 return &bpf_sk_release_proto;
7241 case BPF_FUNC_skc_lookup_tcp:
7242 return &bpf_skc_lookup_tcp_proto;
7243 case BPF_FUNC_tcp_sock:
7244 return &bpf_tcp_sock_proto;
7245 case BPF_FUNC_get_listener_sock:
7246 return &bpf_get_listener_sock_proto;
7247 case BPF_FUNC_skb_ecn_set_ce:
7248 return &bpf_skb_ecn_set_ce_proto;
7249#endif
7250 default:
7251 return sk_filter_func_proto(func_id, prog);
7252 }
7253}
7254
7255static const struct bpf_func_proto *
7256tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7257{
7258 switch (func_id) {
7259 case BPF_FUNC_skb_store_bytes:
7260 return &bpf_skb_store_bytes_proto;
7261 case BPF_FUNC_skb_load_bytes:
7262 return &bpf_skb_load_bytes_proto;
7263 case BPF_FUNC_skb_load_bytes_relative:
7264 return &bpf_skb_load_bytes_relative_proto;
7265 case BPF_FUNC_skb_pull_data:
7266 return &bpf_skb_pull_data_proto;
7267 case BPF_FUNC_csum_diff:
7268 return &bpf_csum_diff_proto;
7269 case BPF_FUNC_csum_update:
7270 return &bpf_csum_update_proto;
7271 case BPF_FUNC_csum_level:
7272 return &bpf_csum_level_proto;
7273 case BPF_FUNC_l3_csum_replace:
7274 return &bpf_l3_csum_replace_proto;
7275 case BPF_FUNC_l4_csum_replace:
7276 return &bpf_l4_csum_replace_proto;
7277 case BPF_FUNC_clone_redirect:
7278 return &bpf_clone_redirect_proto;
7279 case BPF_FUNC_get_cgroup_classid:
7280 return &bpf_get_cgroup_classid_proto;
7281 case BPF_FUNC_skb_vlan_push:
7282 return &bpf_skb_vlan_push_proto;
7283 case BPF_FUNC_skb_vlan_pop:
7284 return &bpf_skb_vlan_pop_proto;
7285 case BPF_FUNC_skb_change_proto:
7286 return &bpf_skb_change_proto_proto;
7287 case BPF_FUNC_skb_change_type:
7288 return &bpf_skb_change_type_proto;
7289 case BPF_FUNC_skb_adjust_room:
7290 return &bpf_skb_adjust_room_proto;
7291 case BPF_FUNC_skb_change_tail:
7292 return &bpf_skb_change_tail_proto;
7293 case BPF_FUNC_skb_change_head:
7294 return &bpf_skb_change_head_proto;
7295 case BPF_FUNC_skb_get_tunnel_key:
7296 return &bpf_skb_get_tunnel_key_proto;
7297 case BPF_FUNC_skb_set_tunnel_key:
7298 return bpf_get_skb_set_tunnel_proto(func_id);
7299 case BPF_FUNC_skb_get_tunnel_opt:
7300 return &bpf_skb_get_tunnel_opt_proto;
7301 case BPF_FUNC_skb_set_tunnel_opt:
7302 return bpf_get_skb_set_tunnel_proto(func_id);
7303 case BPF_FUNC_redirect:
7304 return &bpf_redirect_proto;
7305 case BPF_FUNC_redirect_neigh:
7306 return &bpf_redirect_neigh_proto;
7307 case BPF_FUNC_redirect_peer:
7308 return &bpf_redirect_peer_proto;
7309 case BPF_FUNC_get_route_realm:
7310 return &bpf_get_route_realm_proto;
7311 case BPF_FUNC_get_hash_recalc:
7312 return &bpf_get_hash_recalc_proto;
7313 case BPF_FUNC_set_hash_invalid:
7314 return &bpf_set_hash_invalid_proto;
7315 case BPF_FUNC_set_hash:
7316 return &bpf_set_hash_proto;
7317 case BPF_FUNC_perf_event_output:
7318 return &bpf_skb_event_output_proto;
7319 case BPF_FUNC_get_smp_processor_id:
7320 return &bpf_get_smp_processor_id_proto;
7321 case BPF_FUNC_skb_under_cgroup:
7322 return &bpf_skb_under_cgroup_proto;
7323 case BPF_FUNC_get_socket_cookie:
7324 return &bpf_get_socket_cookie_proto;
7325 case BPF_FUNC_get_socket_uid:
7326 return &bpf_get_socket_uid_proto;
7327 case BPF_FUNC_fib_lookup:
7328 return &bpf_skb_fib_lookup_proto;
7329 case BPF_FUNC_check_mtu:
7330 return &bpf_skb_check_mtu_proto;
7331 case BPF_FUNC_sk_fullsock:
7332 return &bpf_sk_fullsock_proto;
7333 case BPF_FUNC_sk_storage_get:
7334 return &bpf_sk_storage_get_proto;
7335 case BPF_FUNC_sk_storage_delete:
7336 return &bpf_sk_storage_delete_proto;
7337#ifdef CONFIG_XFRM
7338 case BPF_FUNC_skb_get_xfrm_state:
7339 return &bpf_skb_get_xfrm_state_proto;
7340#endif
7341#ifdef CONFIG_CGROUP_NET_CLASSID
7342 case BPF_FUNC_skb_cgroup_classid:
7343 return &bpf_skb_cgroup_classid_proto;
7344#endif
7345#ifdef CONFIG_SOCK_CGROUP_DATA
7346 case BPF_FUNC_skb_cgroup_id:
7347 return &bpf_skb_cgroup_id_proto;
7348 case BPF_FUNC_skb_ancestor_cgroup_id:
7349 return &bpf_skb_ancestor_cgroup_id_proto;
7350#endif
7351#ifdef CONFIG_INET
7352 case BPF_FUNC_sk_lookup_tcp:
7353 return &bpf_sk_lookup_tcp_proto;
7354 case BPF_FUNC_sk_lookup_udp:
7355 return &bpf_sk_lookup_udp_proto;
7356 case BPF_FUNC_sk_release:
7357 return &bpf_sk_release_proto;
7358 case BPF_FUNC_tcp_sock:
7359 return &bpf_tcp_sock_proto;
7360 case BPF_FUNC_get_listener_sock:
7361 return &bpf_get_listener_sock_proto;
7362 case BPF_FUNC_skc_lookup_tcp:
7363 return &bpf_skc_lookup_tcp_proto;
7364 case BPF_FUNC_tcp_check_syncookie:
7365 return &bpf_tcp_check_syncookie_proto;
7366 case BPF_FUNC_skb_ecn_set_ce:
7367 return &bpf_skb_ecn_set_ce_proto;
7368 case BPF_FUNC_tcp_gen_syncookie:
7369 return &bpf_tcp_gen_syncookie_proto;
7370 case BPF_FUNC_sk_assign:
7371 return &bpf_sk_assign_proto;
7372#endif
7373 default:
7374 return bpf_sk_base_func_proto(func_id);
7375 }
7376}
7377
7378static const struct bpf_func_proto *
7379xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7380{
7381 switch (func_id) {
7382 case BPF_FUNC_perf_event_output:
7383 return &bpf_xdp_event_output_proto;
7384 case BPF_FUNC_get_smp_processor_id:
7385 return &bpf_get_smp_processor_id_proto;
7386 case BPF_FUNC_csum_diff:
7387 return &bpf_csum_diff_proto;
7388 case BPF_FUNC_xdp_adjust_head:
7389 return &bpf_xdp_adjust_head_proto;
7390 case BPF_FUNC_xdp_adjust_meta:
7391 return &bpf_xdp_adjust_meta_proto;
7392 case BPF_FUNC_redirect:
7393 return &bpf_xdp_redirect_proto;
7394 case BPF_FUNC_redirect_map:
7395 return &bpf_xdp_redirect_map_proto;
7396 case BPF_FUNC_xdp_adjust_tail:
7397 return &bpf_xdp_adjust_tail_proto;
7398 case BPF_FUNC_fib_lookup:
7399 return &bpf_xdp_fib_lookup_proto;
7400 case BPF_FUNC_check_mtu:
7401 return &bpf_xdp_check_mtu_proto;
7402#ifdef CONFIG_INET
7403 case BPF_FUNC_sk_lookup_udp:
7404 return &bpf_xdp_sk_lookup_udp_proto;
7405 case BPF_FUNC_sk_lookup_tcp:
7406 return &bpf_xdp_sk_lookup_tcp_proto;
7407 case BPF_FUNC_sk_release:
7408 return &bpf_sk_release_proto;
7409 case BPF_FUNC_skc_lookup_tcp:
7410 return &bpf_xdp_skc_lookup_tcp_proto;
7411 case BPF_FUNC_tcp_check_syncookie:
7412 return &bpf_tcp_check_syncookie_proto;
7413 case BPF_FUNC_tcp_gen_syncookie:
7414 return &bpf_tcp_gen_syncookie_proto;
7415#endif
7416 default:
7417 return bpf_sk_base_func_proto(func_id);
7418 }
7419}
7420
7421const struct bpf_func_proto bpf_sock_map_update_proto __weak;
7422const struct bpf_func_proto bpf_sock_hash_update_proto __weak;
7423
7424static const struct bpf_func_proto *
7425sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7426{
7427 switch (func_id) {
7428 case BPF_FUNC_setsockopt:
7429 return &bpf_sock_ops_setsockopt_proto;
7430 case BPF_FUNC_getsockopt:
7431 return &bpf_sock_ops_getsockopt_proto;
7432 case BPF_FUNC_sock_ops_cb_flags_set:
7433 return &bpf_sock_ops_cb_flags_set_proto;
7434 case BPF_FUNC_sock_map_update:
7435 return &bpf_sock_map_update_proto;
7436 case BPF_FUNC_sock_hash_update:
7437 return &bpf_sock_hash_update_proto;
7438 case BPF_FUNC_get_socket_cookie:
7439 return &bpf_get_socket_cookie_sock_ops_proto;
7440 case BPF_FUNC_get_local_storage:
7441 return &bpf_get_local_storage_proto;
7442 case BPF_FUNC_perf_event_output:
7443 return &bpf_event_output_data_proto;
7444 case BPF_FUNC_sk_storage_get:
7445 return &bpf_sk_storage_get_proto;
7446 case BPF_FUNC_sk_storage_delete:
7447 return &bpf_sk_storage_delete_proto;
7448#ifdef CONFIG_INET
7449 case BPF_FUNC_load_hdr_opt:
7450 return &bpf_sock_ops_load_hdr_opt_proto;
7451 case BPF_FUNC_store_hdr_opt:
7452 return &bpf_sock_ops_store_hdr_opt_proto;
7453 case BPF_FUNC_reserve_hdr_opt:
7454 return &bpf_sock_ops_reserve_hdr_opt_proto;
7455 case BPF_FUNC_tcp_sock:
7456 return &bpf_tcp_sock_proto;
7457#endif
7458 default:
7459 return bpf_sk_base_func_proto(func_id);
7460 }
7461}
7462
7463const struct bpf_func_proto bpf_msg_redirect_map_proto __weak;
7464const struct bpf_func_proto bpf_msg_redirect_hash_proto __weak;
7465
7466static const struct bpf_func_proto *
7467sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7468{
7469 switch (func_id) {
7470 case BPF_FUNC_msg_redirect_map:
7471 return &bpf_msg_redirect_map_proto;
7472 case BPF_FUNC_msg_redirect_hash:
7473 return &bpf_msg_redirect_hash_proto;
7474 case BPF_FUNC_msg_apply_bytes:
7475 return &bpf_msg_apply_bytes_proto;
7476 case BPF_FUNC_msg_cork_bytes:
7477 return &bpf_msg_cork_bytes_proto;
7478 case BPF_FUNC_msg_pull_data:
7479 return &bpf_msg_pull_data_proto;
7480 case BPF_FUNC_msg_push_data:
7481 return &bpf_msg_push_data_proto;
7482 case BPF_FUNC_msg_pop_data:
7483 return &bpf_msg_pop_data_proto;
7484 case BPF_FUNC_perf_event_output:
7485 return &bpf_event_output_data_proto;
7486 case BPF_FUNC_get_current_uid_gid:
7487 return &bpf_get_current_uid_gid_proto;
7488 case BPF_FUNC_get_current_pid_tgid:
7489 return &bpf_get_current_pid_tgid_proto;
7490 case BPF_FUNC_sk_storage_get:
7491 return &bpf_sk_storage_get_proto;
7492 case BPF_FUNC_sk_storage_delete:
7493 return &bpf_sk_storage_delete_proto;
7494#ifdef CONFIG_CGROUPS
7495 case BPF_FUNC_get_current_cgroup_id:
7496 return &bpf_get_current_cgroup_id_proto;
7497 case BPF_FUNC_get_current_ancestor_cgroup_id:
7498 return &bpf_get_current_ancestor_cgroup_id_proto;
7499#endif
7500#ifdef CONFIG_CGROUP_NET_CLASSID
7501 case BPF_FUNC_get_cgroup_classid:
7502 return &bpf_get_cgroup_classid_curr_proto;
7503#endif
7504 default:
7505 return bpf_sk_base_func_proto(func_id);
7506 }
7507}
7508
7509const struct bpf_func_proto bpf_sk_redirect_map_proto __weak;
7510const struct bpf_func_proto bpf_sk_redirect_hash_proto __weak;
7511
7512static const struct bpf_func_proto *
7513sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7514{
7515 switch (func_id) {
7516 case BPF_FUNC_skb_store_bytes:
7517 return &bpf_skb_store_bytes_proto;
7518 case BPF_FUNC_skb_load_bytes:
7519 return &bpf_skb_load_bytes_proto;
7520 case BPF_FUNC_skb_pull_data:
7521 return &sk_skb_pull_data_proto;
7522 case BPF_FUNC_skb_change_tail:
7523 return &sk_skb_change_tail_proto;
7524 case BPF_FUNC_skb_change_head:
7525 return &sk_skb_change_head_proto;
7526 case BPF_FUNC_skb_adjust_room:
7527 return &sk_skb_adjust_room_proto;
7528 case BPF_FUNC_get_socket_cookie:
7529 return &bpf_get_socket_cookie_proto;
7530 case BPF_FUNC_get_socket_uid:
7531 return &bpf_get_socket_uid_proto;
7532 case BPF_FUNC_sk_redirect_map:
7533 return &bpf_sk_redirect_map_proto;
7534 case BPF_FUNC_sk_redirect_hash:
7535 return &bpf_sk_redirect_hash_proto;
7536 case BPF_FUNC_perf_event_output:
7537 return &bpf_skb_event_output_proto;
7538#ifdef CONFIG_INET
7539 case BPF_FUNC_sk_lookup_tcp:
7540 return &bpf_sk_lookup_tcp_proto;
7541 case BPF_FUNC_sk_lookup_udp:
7542 return &bpf_sk_lookup_udp_proto;
7543 case BPF_FUNC_sk_release:
7544 return &bpf_sk_release_proto;
7545 case BPF_FUNC_skc_lookup_tcp:
7546 return &bpf_skc_lookup_tcp_proto;
7547#endif
7548 default:
7549 return bpf_sk_base_func_proto(func_id);
7550 }
7551}
7552
7553static const struct bpf_func_proto *
7554flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7555{
7556 switch (func_id) {
7557 case BPF_FUNC_skb_load_bytes:
7558 return &bpf_flow_dissector_load_bytes_proto;
7559 default:
7560 return bpf_sk_base_func_proto(func_id);
7561 }
7562}
7563
7564static const struct bpf_func_proto *
7565lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7566{
7567 switch (func_id) {
7568 case BPF_FUNC_skb_load_bytes:
7569 return &bpf_skb_load_bytes_proto;
7570 case BPF_FUNC_skb_pull_data:
7571 return &bpf_skb_pull_data_proto;
7572 case BPF_FUNC_csum_diff:
7573 return &bpf_csum_diff_proto;
7574 case BPF_FUNC_get_cgroup_classid:
7575 return &bpf_get_cgroup_classid_proto;
7576 case BPF_FUNC_get_route_realm:
7577 return &bpf_get_route_realm_proto;
7578 case BPF_FUNC_get_hash_recalc:
7579 return &bpf_get_hash_recalc_proto;
7580 case BPF_FUNC_perf_event_output:
7581 return &bpf_skb_event_output_proto;
7582 case BPF_FUNC_get_smp_processor_id:
7583 return &bpf_get_smp_processor_id_proto;
7584 case BPF_FUNC_skb_under_cgroup:
7585 return &bpf_skb_under_cgroup_proto;
7586 default:
7587 return bpf_sk_base_func_proto(func_id);
7588 }
7589}
7590
7591static const struct bpf_func_proto *
7592lwt_in_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7593{
7594 switch (func_id) {
7595 case BPF_FUNC_lwt_push_encap:
7596 return &bpf_lwt_in_push_encap_proto;
7597 default:
7598 return lwt_out_func_proto(func_id, prog);
7599 }
7600}
7601
7602static const struct bpf_func_proto *
7603lwt_xmit_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7604{
7605 switch (func_id) {
7606 case BPF_FUNC_skb_get_tunnel_key:
7607 return &bpf_skb_get_tunnel_key_proto;
7608 case BPF_FUNC_skb_set_tunnel_key:
7609 return bpf_get_skb_set_tunnel_proto(func_id);
7610 case BPF_FUNC_skb_get_tunnel_opt:
7611 return &bpf_skb_get_tunnel_opt_proto;
7612 case BPF_FUNC_skb_set_tunnel_opt:
7613 return bpf_get_skb_set_tunnel_proto(func_id);
7614 case BPF_FUNC_redirect:
7615 return &bpf_redirect_proto;
7616 case BPF_FUNC_clone_redirect:
7617 return &bpf_clone_redirect_proto;
7618 case BPF_FUNC_skb_change_tail:
7619 return &bpf_skb_change_tail_proto;
7620 case BPF_FUNC_skb_change_head:
7621 return &bpf_skb_change_head_proto;
7622 case BPF_FUNC_skb_store_bytes:
7623 return &bpf_skb_store_bytes_proto;
7624 case BPF_FUNC_csum_update:
7625 return &bpf_csum_update_proto;
7626 case BPF_FUNC_csum_level:
7627 return &bpf_csum_level_proto;
7628 case BPF_FUNC_l3_csum_replace:
7629 return &bpf_l3_csum_replace_proto;
7630 case BPF_FUNC_l4_csum_replace:
7631 return &bpf_l4_csum_replace_proto;
7632 case BPF_FUNC_set_hash_invalid:
7633 return &bpf_set_hash_invalid_proto;
7634 case BPF_FUNC_lwt_push_encap:
7635 return &bpf_lwt_xmit_push_encap_proto;
7636 default:
7637 return lwt_out_func_proto(func_id, prog);
7638 }
7639}
7640
7641static const struct bpf_func_proto *
7642lwt_seg6local_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7643{
7644 switch (func_id) {
7645#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
7646 case BPF_FUNC_lwt_seg6_store_bytes:
7647 return &bpf_lwt_seg6_store_bytes_proto;
7648 case BPF_FUNC_lwt_seg6_action:
7649 return &bpf_lwt_seg6_action_proto;
7650 case BPF_FUNC_lwt_seg6_adjust_srh:
7651 return &bpf_lwt_seg6_adjust_srh_proto;
7652#endif
7653 default:
7654 return lwt_out_func_proto(func_id, prog);
7655 }
7656}
7657
7658static bool bpf_skb_is_valid_access(int off, int size, enum bpf_access_type type,
7659 const struct bpf_prog *prog,
7660 struct bpf_insn_access_aux *info)
7661{
7662 const int size_default = sizeof(__u32);
7663
7664 if (off < 0 || off >= sizeof(struct __sk_buff))
7665 return false;
7666
7667
7668 if (off % size != 0)
7669 return false;
7670
7671 switch (off) {
7672 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
7673 if (off + size > offsetofend(struct __sk_buff, cb[4]))
7674 return false;
7675 break;
7676 case bpf_ctx_range_till(struct __sk_buff, remote_ip6[0], remote_ip6[3]):
7677 case bpf_ctx_range_till(struct __sk_buff, local_ip6[0], local_ip6[3]):
7678 case bpf_ctx_range_till(struct __sk_buff, remote_ip4, remote_ip4):
7679 case bpf_ctx_range_till(struct __sk_buff, local_ip4, local_ip4):
7680 case bpf_ctx_range(struct __sk_buff, data):
7681 case bpf_ctx_range(struct __sk_buff, data_meta):
7682 case bpf_ctx_range(struct __sk_buff, data_end):
7683 if (size != size_default)
7684 return false;
7685 break;
7686 case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
7687 return false;
7688 case bpf_ctx_range(struct __sk_buff, tstamp):
7689 if (size != sizeof(__u64))
7690 return false;
7691 break;
7692 case offsetof(struct __sk_buff, sk):
7693 if (type == BPF_WRITE || size != sizeof(__u64))
7694 return false;
7695 info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL;
7696 break;
7697 default:
7698
7699 if (type == BPF_WRITE) {
7700 if (size != size_default)
7701 return false;
7702 } else {
7703 bpf_ctx_record_field_size(info, size_default);
7704 if (!bpf_ctx_narrow_access_ok(off, size, size_default))
7705 return false;
7706 }
7707 }
7708
7709 return true;
7710}
7711
7712static bool sk_filter_is_valid_access(int off, int size,
7713 enum bpf_access_type type,
7714 const struct bpf_prog *prog,
7715 struct bpf_insn_access_aux *info)
7716{
7717 switch (off) {
7718 case bpf_ctx_range(struct __sk_buff, tc_classid):
7719 case bpf_ctx_range(struct __sk_buff, data):
7720 case bpf_ctx_range(struct __sk_buff, data_meta):
7721 case bpf_ctx_range(struct __sk_buff, data_end):
7722 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
7723 case bpf_ctx_range(struct __sk_buff, tstamp):
7724 case bpf_ctx_range(struct __sk_buff, wire_len):
7725 return false;
7726 }
7727
7728 if (type == BPF_WRITE) {
7729 switch (off) {
7730 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
7731 break;
7732 default:
7733 return false;
7734 }
7735 }
7736
7737 return bpf_skb_is_valid_access(off, size, type, prog, info);
7738}
7739
7740static bool cg_skb_is_valid_access(int off, int size,
7741 enum bpf_access_type type,
7742 const struct bpf_prog *prog,
7743 struct bpf_insn_access_aux *info)
7744{
7745 switch (off) {
7746 case bpf_ctx_range(struct __sk_buff, tc_classid):
7747 case bpf_ctx_range(struct __sk_buff, data_meta):
7748 case bpf_ctx_range(struct __sk_buff, wire_len):
7749 return false;
7750 case bpf_ctx_range(struct __sk_buff, data):
7751 case bpf_ctx_range(struct __sk_buff, data_end):
7752 if (!bpf_capable())
7753 return false;
7754 break;
7755 }
7756
7757 if (type == BPF_WRITE) {
7758 switch (off) {
7759 case bpf_ctx_range(struct __sk_buff, mark):
7760 case bpf_ctx_range(struct __sk_buff, priority):
7761 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
7762 break;
7763 case bpf_ctx_range(struct __sk_buff, tstamp):
7764 if (!bpf_capable())
7765 return false;
7766 break;
7767 default:
7768 return false;
7769 }
7770 }
7771
7772 switch (off) {
7773 case bpf_ctx_range(struct __sk_buff, data):
7774 info->reg_type = PTR_TO_PACKET;
7775 break;
7776 case bpf_ctx_range(struct __sk_buff, data_end):
7777 info->reg_type = PTR_TO_PACKET_END;
7778 break;
7779 }
7780
7781 return bpf_skb_is_valid_access(off, size, type, prog, info);
7782}
7783
7784static bool lwt_is_valid_access(int off, int size,
7785 enum bpf_access_type type,
7786 const struct bpf_prog *prog,
7787 struct bpf_insn_access_aux *info)
7788{
7789 switch (off) {
7790 case bpf_ctx_range(struct __sk_buff, tc_classid):
7791 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
7792 case bpf_ctx_range(struct __sk_buff, data_meta):
7793 case bpf_ctx_range(struct __sk_buff, tstamp):
7794 case bpf_ctx_range(struct __sk_buff, wire_len):
7795 return false;
7796 }
7797
7798 if (type == BPF_WRITE) {
7799 switch (off) {
7800 case bpf_ctx_range(struct __sk_buff, mark):
7801 case bpf_ctx_range(struct __sk_buff, priority):
7802 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
7803 break;
7804 default:
7805 return false;
7806 }
7807 }
7808
7809 switch (off) {
7810 case bpf_ctx_range(struct __sk_buff, data):
7811 info->reg_type = PTR_TO_PACKET;
7812 break;
7813 case bpf_ctx_range(struct __sk_buff, data_end):
7814 info->reg_type = PTR_TO_PACKET_END;
7815 break;
7816 }
7817
7818 return bpf_skb_is_valid_access(off, size, type, prog, info);
7819}
7820
7821
7822static bool __sock_filter_check_attach_type(int off,
7823 enum bpf_access_type access_type,
7824 enum bpf_attach_type attach_type)
7825{
7826 switch (off) {
7827 case offsetof(struct bpf_sock, bound_dev_if):
7828 case offsetof(struct bpf_sock, mark):
7829 case offsetof(struct bpf_sock, priority):
7830 switch (attach_type) {
7831 case BPF_CGROUP_INET_SOCK_CREATE:
7832 case BPF_CGROUP_INET_SOCK_RELEASE:
7833 goto full_access;
7834 default:
7835 return false;
7836 }
7837 case bpf_ctx_range(struct bpf_sock, src_ip4):
7838 switch (attach_type) {
7839 case BPF_CGROUP_INET4_POST_BIND:
7840 goto read_only;
7841 default:
7842 return false;
7843 }
7844 case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
7845 switch (attach_type) {
7846 case BPF_CGROUP_INET6_POST_BIND:
7847 goto read_only;
7848 default:
7849 return false;
7850 }
7851 case bpf_ctx_range(struct bpf_sock, src_port):
7852 switch (attach_type) {
7853 case BPF_CGROUP_INET4_POST_BIND:
7854 case BPF_CGROUP_INET6_POST_BIND:
7855 goto read_only;
7856 default:
7857 return false;
7858 }
7859 }
7860read_only:
7861 return access_type == BPF_READ;
7862full_access:
7863 return true;
7864}
7865
7866bool bpf_sock_common_is_valid_access(int off, int size,
7867 enum bpf_access_type type,
7868 struct bpf_insn_access_aux *info)
7869{
7870 switch (off) {
7871 case bpf_ctx_range_till(struct bpf_sock, type, priority):
7872 return false;
7873 default:
7874 return bpf_sock_is_valid_access(off, size, type, info);
7875 }
7876}
7877
7878bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
7879 struct bpf_insn_access_aux *info)
7880{
7881 const int size_default = sizeof(__u32);
7882
7883 if (off < 0 || off >= sizeof(struct bpf_sock))
7884 return false;
7885 if (off % size != 0)
7886 return false;
7887
7888 switch (off) {
7889 case offsetof(struct bpf_sock, state):
7890 case offsetof(struct bpf_sock, family):
7891 case offsetof(struct bpf_sock, type):
7892 case offsetof(struct bpf_sock, protocol):
7893 case offsetof(struct bpf_sock, dst_port):
7894 case offsetof(struct bpf_sock, src_port):
7895 case offsetof(struct bpf_sock, rx_queue_mapping):
7896 case bpf_ctx_range(struct bpf_sock, src_ip4):
7897 case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
7898 case bpf_ctx_range(struct bpf_sock, dst_ip4):
7899 case bpf_ctx_range_till(struct bpf_sock, dst_ip6[0], dst_ip6[3]):
7900 bpf_ctx_record_field_size(info, size_default);
7901 return bpf_ctx_narrow_access_ok(off, size, size_default);
7902 }
7903
7904 return size == size_default;
7905}
7906
7907static bool sock_filter_is_valid_access(int off, int size,
7908 enum bpf_access_type type,
7909 const struct bpf_prog *prog,
7910 struct bpf_insn_access_aux *info)
7911{
7912 if (!bpf_sock_is_valid_access(off, size, type, info))
7913 return false;
7914 return __sock_filter_check_attach_type(off, type,
7915 prog->expected_attach_type);
7916}
7917
7918static int bpf_noop_prologue(struct bpf_insn *insn_buf, bool direct_write,
7919 const struct bpf_prog *prog)
7920{
7921
7922
7923
7924 return 0;
7925}
7926
7927static int bpf_unclone_prologue(struct bpf_insn *insn_buf, bool direct_write,
7928 const struct bpf_prog *prog, int drop_verdict)
7929{
7930 struct bpf_insn *insn = insn_buf;
7931
7932 if (!direct_write)
7933 return 0;
7934
7935
7936
7937
7938
7939
7940
7941 *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET());
7942 *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_6, CLONED_MASK);
7943 *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_6, 0, 7);
7944
7945
7946 *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
7947 *insn++ = BPF_ALU64_REG(BPF_XOR, BPF_REG_2, BPF_REG_2);
7948 *insn++ = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
7949 BPF_FUNC_skb_pull_data);
7950
7951
7952
7953
7954 *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2);
7955 *insn++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, drop_verdict);
7956 *insn++ = BPF_EXIT_INSN();
7957
7958
7959 *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
7960
7961 *insn++ = prog->insnsi[0];
7962
7963 return insn - insn_buf;
7964}
7965
7966static int bpf_gen_ld_abs(const struct bpf_insn *orig,
7967 struct bpf_insn *insn_buf)
7968{
7969 bool indirect = BPF_MODE(orig->code) == BPF_IND;
7970 struct bpf_insn *insn = insn_buf;
7971
7972 if (!indirect) {
7973 *insn++ = BPF_MOV64_IMM(BPF_REG_2, orig->imm);
7974 } else {
7975 *insn++ = BPF_MOV64_REG(BPF_REG_2, orig->src_reg);
7976 if (orig->imm)
7977 *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, orig->imm);
7978 }
7979
7980 *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_CTX);
7981
7982 switch (BPF_SIZE(orig->code)) {
7983 case BPF_B:
7984 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8_no_cache);
7985 break;
7986 case BPF_H:
7987 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16_no_cache);
7988 break;
7989 case BPF_W:
7990 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32_no_cache);
7991 break;
7992 }
7993
7994 *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_0, 0, 2);
7995 *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0);
7996 *insn++ = BPF_EXIT_INSN();
7997
7998 return insn - insn_buf;
7999}
8000
8001static int tc_cls_act_prologue(struct bpf_insn *insn_buf, bool direct_write,
8002 const struct bpf_prog *prog)
8003{
8004 return bpf_unclone_prologue(insn_buf, direct_write, prog, TC_ACT_SHOT);
8005}
8006
8007static bool tc_cls_act_is_valid_access(int off, int size,
8008 enum bpf_access_type type,
8009 const struct bpf_prog *prog,
8010 struct bpf_insn_access_aux *info)
8011{
8012 if (type == BPF_WRITE) {
8013 switch (off) {
8014 case bpf_ctx_range(struct __sk_buff, mark):
8015 case bpf_ctx_range(struct __sk_buff, tc_index):
8016 case bpf_ctx_range(struct __sk_buff, priority):
8017 case bpf_ctx_range(struct __sk_buff, tc_classid):
8018 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
8019 case bpf_ctx_range(struct __sk_buff, tstamp):
8020 case bpf_ctx_range(struct __sk_buff, queue_mapping):
8021 break;
8022 default:
8023 return false;
8024 }
8025 }
8026
8027 switch (off) {
8028 case bpf_ctx_range(struct __sk_buff, data):
8029 info->reg_type = PTR_TO_PACKET;
8030 break;
8031 case bpf_ctx_range(struct __sk_buff, data_meta):
8032 info->reg_type = PTR_TO_PACKET_META;
8033 break;
8034 case bpf_ctx_range(struct __sk_buff, data_end):
8035 info->reg_type = PTR_TO_PACKET_END;
8036 break;
8037 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
8038 return false;
8039 }
8040
8041 return bpf_skb_is_valid_access(off, size, type, prog, info);
8042}
8043
8044static bool __is_valid_xdp_access(int off, int size)
8045{
8046 if (off < 0 || off >= sizeof(struct xdp_md))
8047 return false;
8048 if (off % size != 0)
8049 return false;
8050 if (size != sizeof(__u32))
8051 return false;
8052
8053 return true;
8054}
8055
8056static bool xdp_is_valid_access(int off, int size,
8057 enum bpf_access_type type,
8058 const struct bpf_prog *prog,
8059 struct bpf_insn_access_aux *info)
8060{
8061 if (prog->expected_attach_type != BPF_XDP_DEVMAP) {
8062 switch (off) {
8063 case offsetof(struct xdp_md, egress_ifindex):
8064 return false;
8065 }
8066 }
8067
8068 if (type == BPF_WRITE) {
8069 if (bpf_prog_is_dev_bound(prog->aux)) {
8070 switch (off) {
8071 case offsetof(struct xdp_md, rx_queue_index):
8072 return __is_valid_xdp_access(off, size);
8073 }
8074 }
8075 return false;
8076 }
8077
8078 switch (off) {
8079 case offsetof(struct xdp_md, data):
8080 info->reg_type = PTR_TO_PACKET;
8081 break;
8082 case offsetof(struct xdp_md, data_meta):
8083 info->reg_type = PTR_TO_PACKET_META;
8084 break;
8085 case offsetof(struct xdp_md, data_end):
8086 info->reg_type = PTR_TO_PACKET_END;
8087 break;
8088 }
8089
8090 return __is_valid_xdp_access(off, size);
8091}
8092
8093void bpf_warn_invalid_xdp_action(u32 act)
8094{
8095 const u32 act_max = XDP_REDIRECT;
8096
8097 WARN_ONCE(1, "%s XDP return value %u, expect packet loss!\n",
8098 act > act_max ? "Illegal" : "Driver unsupported",
8099 act);
8100}
8101EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action);
8102
8103static bool sock_addr_is_valid_access(int off, int size,
8104 enum bpf_access_type type,
8105 const struct bpf_prog *prog,
8106 struct bpf_insn_access_aux *info)
8107{
8108 const int size_default = sizeof(__u32);
8109
8110 if (off < 0 || off >= sizeof(struct bpf_sock_addr))
8111 return false;
8112 if (off % size != 0)
8113 return false;
8114
8115
8116
8117
8118 switch (off) {
8119 case bpf_ctx_range(struct bpf_sock_addr, user_ip4):
8120 switch (prog->expected_attach_type) {
8121 case BPF_CGROUP_INET4_BIND:
8122 case BPF_CGROUP_INET4_CONNECT:
8123 case BPF_CGROUP_INET4_GETPEERNAME:
8124 case BPF_CGROUP_INET4_GETSOCKNAME:
8125 case BPF_CGROUP_UDP4_SENDMSG:
8126 case BPF_CGROUP_UDP4_RECVMSG:
8127 break;
8128 default:
8129 return false;
8130 }
8131 break;
8132 case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
8133 switch (prog->expected_attach_type) {
8134 case BPF_CGROUP_INET6_BIND:
8135 case BPF_CGROUP_INET6_CONNECT:
8136 case BPF_CGROUP_INET6_GETPEERNAME:
8137 case BPF_CGROUP_INET6_GETSOCKNAME:
8138 case BPF_CGROUP_UDP6_SENDMSG:
8139 case BPF_CGROUP_UDP6_RECVMSG:
8140 break;
8141 default:
8142 return false;
8143 }
8144 break;
8145 case bpf_ctx_range(struct bpf_sock_addr, msg_src_ip4):
8146 switch (prog->expected_attach_type) {
8147 case BPF_CGROUP_UDP4_SENDMSG:
8148 break;
8149 default:
8150 return false;
8151 }
8152 break;
8153 case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
8154 msg_src_ip6[3]):
8155 switch (prog->expected_attach_type) {
8156 case BPF_CGROUP_UDP6_SENDMSG:
8157 break;
8158 default:
8159 return false;
8160 }
8161 break;
8162 }
8163
8164 switch (off) {
8165 case bpf_ctx_range(struct bpf_sock_addr, user_ip4):
8166 case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
8167 case bpf_ctx_range(struct bpf_sock_addr, msg_src_ip4):
8168 case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
8169 msg_src_ip6[3]):
8170 case bpf_ctx_range(struct bpf_sock_addr, user_port):
8171 if (type == BPF_READ) {
8172 bpf_ctx_record_field_size(info, size_default);
8173
8174 if (bpf_ctx_wide_access_ok(off, size,
8175 struct bpf_sock_addr,
8176 user_ip6))
8177 return true;
8178
8179 if (bpf_ctx_wide_access_ok(off, size,
8180 struct bpf_sock_addr,
8181 msg_src_ip6))
8182 return true;
8183
8184 if (!bpf_ctx_narrow_access_ok(off, size, size_default))
8185 return false;
8186 } else {
8187 if (bpf_ctx_wide_access_ok(off, size,
8188 struct bpf_sock_addr,
8189 user_ip6))
8190 return true;
8191
8192 if (bpf_ctx_wide_access_ok(off, size,
8193 struct bpf_sock_addr,
8194 msg_src_ip6))
8195 return true;
8196
8197 if (size != size_default)
8198 return false;
8199 }
8200 break;
8201 case offsetof(struct bpf_sock_addr, sk):
8202 if (type != BPF_READ)
8203 return false;
8204 if (size != sizeof(__u64))
8205 return false;
8206 info->reg_type = PTR_TO_SOCKET;
8207 break;
8208 default:
8209 if (type == BPF_READ) {
8210 if (size != size_default)
8211 return false;
8212 } else {
8213 return false;
8214 }
8215 }
8216
8217 return true;
8218}
8219
8220static bool sock_ops_is_valid_access(int off, int size,
8221 enum bpf_access_type type,
8222 const struct bpf_prog *prog,
8223 struct bpf_insn_access_aux *info)
8224{
8225 const int size_default = sizeof(__u32);
8226
8227 if (off < 0 || off >= sizeof(struct bpf_sock_ops))
8228 return false;
8229
8230
8231 if (off % size != 0)
8232 return false;
8233
8234 if (type == BPF_WRITE) {
8235 switch (off) {
8236 case offsetof(struct bpf_sock_ops, reply):
8237 case offsetof(struct bpf_sock_ops, sk_txhash):
8238 if (size != size_default)
8239 return false;
8240 break;
8241 default:
8242 return false;
8243 }
8244 } else {
8245 switch (off) {
8246 case bpf_ctx_range_till(struct bpf_sock_ops, bytes_received,
8247 bytes_acked):
8248 if (size != sizeof(__u64))
8249 return false;
8250 break;
8251 case offsetof(struct bpf_sock_ops, sk):
8252 if (size != sizeof(__u64))
8253 return false;
8254 info->reg_type = PTR_TO_SOCKET_OR_NULL;
8255 break;
8256 case offsetof(struct bpf_sock_ops, skb_data):
8257 if (size != sizeof(__u64))
8258 return false;
8259 info->reg_type = PTR_TO_PACKET;
8260 break;
8261 case offsetof(struct bpf_sock_ops, skb_data_end):
8262 if (size != sizeof(__u64))
8263 return false;
8264 info->reg_type = PTR_TO_PACKET_END;
8265 break;
8266 case offsetof(struct bpf_sock_ops, skb_tcp_flags):
8267 bpf_ctx_record_field_size(info, size_default);
8268 return bpf_ctx_narrow_access_ok(off, size,
8269 size_default);
8270 default:
8271 if (size != size_default)
8272 return false;
8273 break;
8274 }
8275 }
8276
8277 return true;
8278}
8279
8280static int sk_skb_prologue(struct bpf_insn *insn_buf, bool direct_write,
8281 const struct bpf_prog *prog)
8282{
8283 return bpf_unclone_prologue(insn_buf, direct_write, prog, SK_DROP);
8284}
8285
8286static bool sk_skb_is_valid_access(int off, int size,
8287 enum bpf_access_type type,
8288 const struct bpf_prog *prog,
8289 struct bpf_insn_access_aux *info)
8290{
8291 switch (off) {
8292 case bpf_ctx_range(struct __sk_buff, tc_classid):
8293 case bpf_ctx_range(struct __sk_buff, data_meta):
8294 case bpf_ctx_range(struct __sk_buff, tstamp):
8295 case bpf_ctx_range(struct __sk_buff, wire_len):
8296 return false;
8297 }
8298
8299 if (type == BPF_WRITE) {
8300 switch (off) {
8301 case bpf_ctx_range(struct __sk_buff, tc_index):
8302 case bpf_ctx_range(struct __sk_buff, priority):
8303 break;
8304 default:
8305 return false;
8306 }
8307 }
8308
8309 switch (off) {
8310 case bpf_ctx_range(struct __sk_buff, mark):
8311 return false;
8312 case bpf_ctx_range(struct __sk_buff, data):
8313 info->reg_type = PTR_TO_PACKET;
8314 break;
8315 case bpf_ctx_range(struct __sk_buff, data_end):
8316 info->reg_type = PTR_TO_PACKET_END;
8317 break;
8318 }
8319
8320 return bpf_skb_is_valid_access(off, size, type, prog, info);
8321}
8322
8323static bool sk_msg_is_valid_access(int off, int size,
8324 enum bpf_access_type type,
8325 const struct bpf_prog *prog,
8326 struct bpf_insn_access_aux *info)
8327{
8328 if (type == BPF_WRITE)
8329 return false;
8330
8331 if (off % size != 0)
8332 return false;
8333
8334 switch (off) {
8335 case offsetof(struct sk_msg_md, data):
8336 info->reg_type = PTR_TO_PACKET;
8337 if (size != sizeof(__u64))
8338 return false;
8339 break;
8340 case offsetof(struct sk_msg_md, data_end):
8341 info->reg_type = PTR_TO_PACKET_END;
8342 if (size != sizeof(__u64))
8343 return false;
8344 break;
8345 case offsetof(struct sk_msg_md, sk):
8346 if (size != sizeof(__u64))
8347 return false;
8348 info->reg_type = PTR_TO_SOCKET;
8349 break;
8350 case bpf_ctx_range(struct sk_msg_md, family):
8351 case bpf_ctx_range(struct sk_msg_md, remote_ip4):
8352 case bpf_ctx_range(struct sk_msg_md, local_ip4):
8353 case bpf_ctx_range_till(struct sk_msg_md, remote_ip6[0], remote_ip6[3]):
8354 case bpf_ctx_range_till(struct sk_msg_md, local_ip6[0], local_ip6[3]):
8355 case bpf_ctx_range(struct sk_msg_md, remote_port):
8356 case bpf_ctx_range(struct sk_msg_md, local_port):
8357 case bpf_ctx_range(struct sk_msg_md, size):
8358 if (size != sizeof(__u32))
8359 return false;
8360 break;
8361 default:
8362 return false;
8363 }
8364 return true;
8365}
8366
8367static bool flow_dissector_is_valid_access(int off, int size,
8368 enum bpf_access_type type,
8369 const struct bpf_prog *prog,
8370 struct bpf_insn_access_aux *info)
8371{
8372 const int size_default = sizeof(__u32);
8373
8374 if (off < 0 || off >= sizeof(struct __sk_buff))
8375 return false;
8376
8377 if (type == BPF_WRITE)
8378 return false;
8379
8380 switch (off) {
8381 case bpf_ctx_range(struct __sk_buff, data):
8382 if (size != size_default)
8383 return false;
8384 info->reg_type = PTR_TO_PACKET;
8385 return true;
8386 case bpf_ctx_range(struct __sk_buff, data_end):
8387 if (size != size_default)
8388 return false;
8389 info->reg_type = PTR_TO_PACKET_END;
8390 return true;
8391 case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
8392 if (size != sizeof(__u64))
8393 return false;
8394 info->reg_type = PTR_TO_FLOW_KEYS;
8395 return true;
8396 default:
8397 return false;
8398 }
8399}
8400
8401static u32 flow_dissector_convert_ctx_access(enum bpf_access_type type,
8402 const struct bpf_insn *si,
8403 struct bpf_insn *insn_buf,
8404 struct bpf_prog *prog,
8405 u32 *target_size)
8406
8407{
8408 struct bpf_insn *insn = insn_buf;
8409
8410 switch (si->off) {
8411 case offsetof(struct __sk_buff, data):
8412 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, data),
8413 si->dst_reg, si->src_reg,
8414 offsetof(struct bpf_flow_dissector, data));
8415 break;
8416
8417 case offsetof(struct __sk_buff, data_end):
8418 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, data_end),
8419 si->dst_reg, si->src_reg,
8420 offsetof(struct bpf_flow_dissector, data_end));
8421 break;
8422
8423 case offsetof(struct __sk_buff, flow_keys):
8424 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, flow_keys),
8425 si->dst_reg, si->src_reg,
8426 offsetof(struct bpf_flow_dissector, flow_keys));
8427 break;
8428 }
8429
8430 return insn - insn_buf;
8431}
8432
8433static struct bpf_insn *bpf_convert_shinfo_access(const struct bpf_insn *si,
8434 struct bpf_insn *insn)
8435{
8436
8437#ifdef NET_SKBUFF_DATA_USES_OFFSET
8438 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end),
8439 BPF_REG_AX, si->src_reg,
8440 offsetof(struct sk_buff, end));
8441 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, head),
8442 si->dst_reg, si->src_reg,
8443 offsetof(struct sk_buff, head));
8444 *insn++ = BPF_ALU64_REG(BPF_ADD, si->dst_reg, BPF_REG_AX);
8445#else
8446 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end),
8447 si->dst_reg, si->src_reg,
8448 offsetof(struct sk_buff, end));
8449#endif
8450
8451 return insn;
8452}
8453
8454static u32 bpf_convert_ctx_access(enum bpf_access_type type,
8455 const struct bpf_insn *si,
8456 struct bpf_insn *insn_buf,
8457 struct bpf_prog *prog, u32 *target_size)
8458{
8459 struct bpf_insn *insn = insn_buf;
8460 int off;
8461
8462 switch (si->off) {
8463 case offsetof(struct __sk_buff, len):
8464 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
8465 bpf_target_off(struct sk_buff, len, 4,
8466 target_size));
8467 break;
8468
8469 case offsetof(struct __sk_buff, protocol):
8470 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
8471 bpf_target_off(struct sk_buff, protocol, 2,
8472 target_size));
8473 break;
8474
8475 case offsetof(struct __sk_buff, vlan_proto):
8476 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
8477 bpf_target_off(struct sk_buff, vlan_proto, 2,
8478 target_size));
8479 break;
8480
8481 case offsetof(struct __sk_buff, priority):
8482 if (type == BPF_WRITE)
8483 *insn++ = BPF_STX_MEM(BPF_W, si->dst_reg, si->src_reg,
8484 bpf_target_off(struct sk_buff, priority, 4,
8485 target_size));
8486 else
8487 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
8488 bpf_target_off(struct sk_buff, priority, 4,
8489 target_size));
8490 break;
8491
8492 case offsetof(struct __sk_buff, ingress_ifindex):
8493 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
8494 bpf_target_off(struct sk_buff, skb_iif, 4,
8495 target_size));
8496 break;
8497
8498 case offsetof(struct __sk_buff, ifindex):
8499 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
8500 si->dst_reg, si->src_reg,
8501 offsetof(struct sk_buff, dev));
8502 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
8503 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
8504 bpf_target_off(struct net_device, ifindex, 4,
8505 target_size));
8506 break;
8507
8508 case offsetof(struct __sk_buff, hash):
8509 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
8510 bpf_target_off(struct sk_buff, hash, 4,
8511 target_size));
8512 break;
8513
8514 case offsetof(struct __sk_buff, mark):
8515 if (type == BPF_WRITE)
8516 *insn++ = BPF_STX_MEM(BPF_W, si->dst_reg, si->src_reg,
8517 bpf_target_off(struct sk_buff, mark, 4,
8518 target_size));
8519 else
8520 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
8521 bpf_target_off(struct sk_buff, mark, 4,
8522 target_size));
8523 break;
8524
8525 case offsetof(struct __sk_buff, pkt_type):
8526 *target_size = 1;
8527 *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg,
8528 PKT_TYPE_OFFSET());
8529 *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, PKT_TYPE_MAX);
8530#ifdef __BIG_ENDIAN_BITFIELD
8531 *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, 5);
8532#endif
8533 break;
8534
8535 case offsetof(struct __sk_buff, queue_mapping):
8536 if (type == BPF_WRITE) {
8537 *insn++ = BPF_JMP_IMM(BPF_JGE, si->src_reg, NO_QUEUE_MAPPING, 1);
8538 *insn++ = BPF_STX_MEM(BPF_H, si->dst_reg, si->src_reg,
8539 bpf_target_off(struct sk_buff,
8540 queue_mapping,
8541 2, target_size));
8542 } else {
8543 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
8544 bpf_target_off(struct sk_buff,
8545 queue_mapping,
8546 2, target_size));
8547 }
8548 break;
8549
8550 case offsetof(struct __sk_buff, vlan_present):
8551 *target_size = 1;
8552 *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg,
8553 PKT_VLAN_PRESENT_OFFSET());
8554 if (PKT_VLAN_PRESENT_BIT)
8555 *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, PKT_VLAN_PRESENT_BIT);
8556 if (PKT_VLAN_PRESENT_BIT < 7)
8557 *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, 1);
8558 break;
8559
8560 case offsetof(struct __sk_buff, vlan_tci):
8561 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
8562 bpf_target_off(struct sk_buff, vlan_tci, 2,
8563 target_size));
8564 break;
8565
8566 case offsetof(struct __sk_buff, cb[0]) ...
8567 offsetofend(struct __sk_buff, cb[4]) - 1:
8568 BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, data) < 20);
8569 BUILD_BUG_ON((offsetof(struct sk_buff, cb) +
8570 offsetof(struct qdisc_skb_cb, data)) %
8571 sizeof(__u64));
8572
8573 prog->cb_access = 1;
8574 off = si->off;
8575 off -= offsetof(struct __sk_buff, cb[0]);
8576 off += offsetof(struct sk_buff, cb);
8577 off += offsetof(struct qdisc_skb_cb, data);
8578 if (type == BPF_WRITE)
8579 *insn++ = BPF_STX_MEM(BPF_SIZE(si->code), si->dst_reg,
8580 si->src_reg, off);
8581 else
8582 *insn++ = BPF_LDX_MEM(BPF_SIZE(si->code), si->dst_reg,
8583 si->src_reg, off);
8584 break;
8585
8586 case offsetof(struct __sk_buff, tc_classid):
8587 BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, tc_classid) != 2);
8588
8589 off = si->off;
8590 off -= offsetof(struct __sk_buff, tc_classid);
8591 off += offsetof(struct sk_buff, cb);
8592 off += offsetof(struct qdisc_skb_cb, tc_classid);
8593 *target_size = 2;
8594 if (type == BPF_WRITE)
8595 *insn++ = BPF_STX_MEM(BPF_H, si->dst_reg,
8596 si->src_reg, off);
8597 else
8598 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg,
8599 si->src_reg, off);
8600 break;
8601
8602 case offsetof(struct __sk_buff, data):
8603 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
8604 si->dst_reg, si->src_reg,
8605 offsetof(struct sk_buff, data));
8606 break;
8607
8608 case offsetof(struct __sk_buff, data_meta):
8609 off = si->off;
8610 off -= offsetof(struct __sk_buff, data_meta);
8611 off += offsetof(struct sk_buff, cb);
8612 off += offsetof(struct bpf_skb_data_end, data_meta);
8613 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg,
8614 si->src_reg, off);
8615 break;
8616
8617 case offsetof(struct __sk_buff, data_end):
8618 off = si->off;
8619 off -= offsetof(struct __sk_buff, data_end);
8620 off += offsetof(struct sk_buff, cb);
8621 off += offsetof(struct bpf_skb_data_end, data_end);
8622 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg,
8623 si->src_reg, off);
8624 break;
8625
8626 case offsetof(struct __sk_buff, tc_index):
8627#ifdef CONFIG_NET_SCHED
8628 if (type == BPF_WRITE)
8629 *insn++ = BPF_STX_MEM(BPF_H, si->dst_reg, si->src_reg,
8630 bpf_target_off(struct sk_buff, tc_index, 2,
8631 target_size));
8632 else
8633 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
8634 bpf_target_off(struct sk_buff, tc_index, 2,
8635 target_size));
8636#else
8637 *target_size = 2;
8638 if (type == BPF_WRITE)
8639 *insn++ = BPF_MOV64_REG(si->dst_reg, si->dst_reg);
8640 else
8641 *insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
8642#endif
8643 break;
8644
8645 case offsetof(struct __sk_buff, napi_id):
8646#if defined(CONFIG_NET_RX_BUSY_POLL)
8647 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
8648 bpf_target_off(struct sk_buff, napi_id, 4,
8649 target_size));
8650 *insn++ = BPF_JMP_IMM(BPF_JGE, si->dst_reg, MIN_NAPI_ID, 1);
8651 *insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
8652#else
8653 *target_size = 4;
8654 *insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
8655#endif
8656 break;
8657 case offsetof(struct __sk_buff, family):
8658 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);
8659
8660 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
8661 si->dst_reg, si->src_reg,
8662 offsetof(struct sk_buff, sk));
8663 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
8664 bpf_target_off(struct sock_common,
8665 skc_family,
8666 2, target_size));
8667 break;
8668 case offsetof(struct __sk_buff, remote_ip4):
8669 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);
8670
8671 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
8672 si->dst_reg, si->src_reg,
8673 offsetof(struct sk_buff, sk));
8674 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
8675 bpf_target_off(struct sock_common,
8676 skc_daddr,
8677 4, target_size));
8678 break;
8679 case offsetof(struct __sk_buff, local_ip4):
8680 BUILD_BUG_ON(sizeof_field(struct sock_common,
8681 skc_rcv_saddr) != 4);
8682
8683 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
8684 si->dst_reg, si->src_reg,
8685 offsetof(struct sk_buff, sk));
8686 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
8687 bpf_target_off(struct sock_common,
8688 skc_rcv_saddr,
8689 4, target_size));
8690 break;
8691 case offsetof(struct __sk_buff, remote_ip6[0]) ...
8692 offsetof(struct __sk_buff, remote_ip6[3]):
8693#if IS_ENABLED(CONFIG_IPV6)
8694 BUILD_BUG_ON(sizeof_field(struct sock_common,
8695 skc_v6_daddr.s6_addr32[0]) != 4);
8696
8697 off = si->off;
8698 off -= offsetof(struct __sk_buff, remote_ip6[0]);
8699
8700 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
8701 si->dst_reg, si->src_reg,
8702 offsetof(struct sk_buff, sk));
8703 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
8704 offsetof(struct sock_common,
8705 skc_v6_daddr.s6_addr32[0]) +
8706 off);
8707#else
8708 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
8709#endif
8710 break;
8711 case offsetof(struct __sk_buff, local_ip6[0]) ...
8712 offsetof(struct __sk_buff, local_ip6[3]):
8713#if IS_ENABLED(CONFIG_IPV6)
8714 BUILD_BUG_ON(sizeof_field(struct sock_common,
8715 skc_v6_rcv_saddr.s6_addr32[0]) != 4);
8716
8717 off = si->off;
8718 off -= offsetof(struct __sk_buff, local_ip6[0]);
8719
8720 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
8721 si->dst_reg, si->src_reg,
8722 offsetof(struct sk_buff, sk));
8723 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
8724 offsetof(struct sock_common,
8725 skc_v6_rcv_saddr.s6_addr32[0]) +
8726 off);
8727#else
8728 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
8729#endif
8730 break;
8731
8732 case offsetof(struct __sk_buff, remote_port):
8733 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);
8734
8735 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
8736 si->dst_reg, si->src_reg,
8737 offsetof(struct sk_buff, sk));
8738 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
8739 bpf_target_off(struct sock_common,
8740 skc_dport,
8741 2, target_size));
8742#ifndef __BIG_ENDIAN_BITFIELD
8743 *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
8744#endif
8745 break;
8746
8747 case offsetof(struct __sk_buff, local_port):
8748 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);
8749
8750 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
8751 si->dst_reg, si->src_reg,
8752 offsetof(struct sk_buff, sk));
8753 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
8754 bpf_target_off(struct sock_common,
8755 skc_num, 2, target_size));
8756 break;
8757
8758 case offsetof(struct __sk_buff, tstamp):
8759 BUILD_BUG_ON(sizeof_field(struct sk_buff, tstamp) != 8);
8760
8761 if (type == BPF_WRITE)
8762 *insn++ = BPF_STX_MEM(BPF_DW,
8763 si->dst_reg, si->src_reg,
8764 bpf_target_off(struct sk_buff,
8765 tstamp, 8,
8766 target_size));
8767 else
8768 *insn++ = BPF_LDX_MEM(BPF_DW,
8769 si->dst_reg, si->src_reg,
8770 bpf_target_off(struct sk_buff,
8771 tstamp, 8,
8772 target_size));
8773 break;
8774
8775 case offsetof(struct __sk_buff, gso_segs):
8776 insn = bpf_convert_shinfo_access(si, insn);
8777 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct skb_shared_info, gso_segs),
8778 si->dst_reg, si->dst_reg,
8779 bpf_target_off(struct skb_shared_info,
8780 gso_segs, 2,
8781 target_size));
8782 break;
8783 case offsetof(struct __sk_buff, gso_size):
8784 insn = bpf_convert_shinfo_access(si, insn);
8785 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct skb_shared_info, gso_size),
8786 si->dst_reg, si->dst_reg,
8787 bpf_target_off(struct skb_shared_info,
8788 gso_size, 2,
8789 target_size));
8790 break;
8791 case offsetof(struct __sk_buff, wire_len):
8792 BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, pkt_len) != 4);
8793
8794 off = si->off;
8795 off -= offsetof(struct __sk_buff, wire_len);
8796 off += offsetof(struct sk_buff, cb);
8797 off += offsetof(struct qdisc_skb_cb, pkt_len);
8798 *target_size = 4;
8799 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, off);
8800 break;
8801
8802 case offsetof(struct __sk_buff, sk):
8803 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
8804 si->dst_reg, si->src_reg,
8805 offsetof(struct sk_buff, sk));
8806 break;
8807 }
8808
8809 return insn - insn_buf;
8810}
8811
8812u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
8813 const struct bpf_insn *si,
8814 struct bpf_insn *insn_buf,
8815 struct bpf_prog *prog, u32 *target_size)
8816{
8817 struct bpf_insn *insn = insn_buf;
8818 int off;
8819
8820 switch (si->off) {
8821 case offsetof(struct bpf_sock, bound_dev_if):
8822 BUILD_BUG_ON(sizeof_field(struct sock, sk_bound_dev_if) != 4);
8823
8824 if (type == BPF_WRITE)
8825 *insn++ = BPF_STX_MEM(BPF_W, si->dst_reg, si->src_reg,
8826 offsetof(struct sock, sk_bound_dev_if));
8827 else
8828 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
8829 offsetof(struct sock, sk_bound_dev_if));
8830 break;
8831
8832 case offsetof(struct bpf_sock, mark):
8833 BUILD_BUG_ON(sizeof_field(struct sock, sk_mark) != 4);
8834
8835 if (type == BPF_WRITE)
8836 *insn++ = BPF_STX_MEM(BPF_W, si->dst_reg, si->src_reg,
8837 offsetof(struct sock, sk_mark));
8838 else
8839 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
8840 offsetof(struct sock, sk_mark));
8841 break;
8842
8843 case offsetof(struct bpf_sock, priority):
8844 BUILD_BUG_ON(sizeof_field(struct sock, sk_priority) != 4);
8845
8846 if (type == BPF_WRITE)
8847 *insn++ = BPF_STX_MEM(BPF_W, si->dst_reg, si->src_reg,
8848 offsetof(struct sock, sk_priority));
8849 else
8850 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
8851 offsetof(struct sock, sk_priority));
8852 break;
8853
8854 case offsetof(struct bpf_sock, family):
8855 *insn++ = BPF_LDX_MEM(
8856 BPF_FIELD_SIZEOF(struct sock_common, skc_family),
8857 si->dst_reg, si->src_reg,
8858 bpf_target_off(struct sock_common,
8859 skc_family,
8860 sizeof_field(struct sock_common,
8861 skc_family),
8862 target_size));
8863 break;
8864
8865 case offsetof(struct bpf_sock, type):
8866 *insn++ = BPF_LDX_MEM(
8867 BPF_FIELD_SIZEOF(struct sock, sk_type),
8868 si->dst_reg, si->src_reg,
8869 bpf_target_off(struct sock, sk_type,
8870 sizeof_field(struct sock, sk_type),
8871 target_size));
8872 break;
8873
8874 case offsetof(struct bpf_sock, protocol):
8875 *insn++ = BPF_LDX_MEM(
8876 BPF_FIELD_SIZEOF(struct sock, sk_protocol),
8877 si->dst_reg, si->src_reg,
8878 bpf_target_off(struct sock, sk_protocol,
8879 sizeof_field(struct sock, sk_protocol),
8880 target_size));
8881 break;
8882
8883 case offsetof(struct bpf_sock, src_ip4):
8884 *insn++ = BPF_LDX_MEM(
8885 BPF_SIZE(si->code), si->dst_reg, si->src_reg,
8886 bpf_target_off(struct sock_common, skc_rcv_saddr,
8887 sizeof_field(struct sock_common,
8888 skc_rcv_saddr),
8889 target_size));
8890 break;
8891
8892 case offsetof(struct bpf_sock, dst_ip4):
8893 *insn++ = BPF_LDX_MEM(
8894 BPF_SIZE(si->code), si->dst_reg, si->src_reg,
8895 bpf_target_off(struct sock_common, skc_daddr,
8896 sizeof_field(struct sock_common,
8897 skc_daddr),
8898 target_size));
8899 break;
8900
8901 case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
8902#if IS_ENABLED(CONFIG_IPV6)
8903 off = si->off;
8904 off -= offsetof(struct bpf_sock, src_ip6[0]);
8905 *insn++ = BPF_LDX_MEM(
8906 BPF_SIZE(si->code), si->dst_reg, si->src_reg,
8907 bpf_target_off(
8908 struct sock_common,
8909 skc_v6_rcv_saddr.s6_addr32[0],
8910 sizeof_field(struct sock_common,
8911 skc_v6_rcv_saddr.s6_addr32[0]),
8912 target_size) + off);
8913#else
8914 (void)off;
8915 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
8916#endif
8917 break;
8918
8919 case bpf_ctx_range_till(struct bpf_sock, dst_ip6[0], dst_ip6[3]):
8920#if IS_ENABLED(CONFIG_IPV6)
8921 off = si->off;
8922 off -= offsetof(struct bpf_sock, dst_ip6[0]);
8923 *insn++ = BPF_LDX_MEM(
8924 BPF_SIZE(si->code), si->dst_reg, si->src_reg,
8925 bpf_target_off(struct sock_common,
8926 skc_v6_daddr.s6_addr32[0],
8927 sizeof_field(struct sock_common,
8928 skc_v6_daddr.s6_addr32[0]),
8929 target_size) + off);
8930#else
8931 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
8932 *target_size = 4;
8933#endif
8934 break;
8935
8936 case offsetof(struct bpf_sock, src_port):
8937 *insn++ = BPF_LDX_MEM(
8938 BPF_FIELD_SIZEOF(struct sock_common, skc_num),
8939 si->dst_reg, si->src_reg,
8940 bpf_target_off(struct sock_common, skc_num,
8941 sizeof_field(struct sock_common,
8942 skc_num),
8943 target_size));
8944 break;
8945
8946 case offsetof(struct bpf_sock, dst_port):
8947 *insn++ = BPF_LDX_MEM(
8948 BPF_FIELD_SIZEOF(struct sock_common, skc_dport),
8949 si->dst_reg, si->src_reg,
8950 bpf_target_off(struct sock_common, skc_dport,
8951 sizeof_field(struct sock_common,
8952 skc_dport),
8953 target_size));
8954 break;
8955
8956 case offsetof(struct bpf_sock, state):
8957 *insn++ = BPF_LDX_MEM(
8958 BPF_FIELD_SIZEOF(struct sock_common, skc_state),
8959 si->dst_reg, si->src_reg,
8960 bpf_target_off(struct sock_common, skc_state,
8961 sizeof_field(struct sock_common,
8962 skc_state),
8963 target_size));
8964 break;
8965 case offsetof(struct bpf_sock, rx_queue_mapping):
8966#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
8967 *insn++ = BPF_LDX_MEM(
8968 BPF_FIELD_SIZEOF(struct sock, sk_rx_queue_mapping),
8969 si->dst_reg, si->src_reg,
8970 bpf_target_off(struct sock, sk_rx_queue_mapping,
8971 sizeof_field(struct sock,
8972 sk_rx_queue_mapping),
8973 target_size));
8974 *insn++ = BPF_JMP_IMM(BPF_JNE, si->dst_reg, NO_QUEUE_MAPPING,
8975 1);
8976 *insn++ = BPF_MOV64_IMM(si->dst_reg, -1);
8977#else
8978 *insn++ = BPF_MOV64_IMM(si->dst_reg, -1);
8979 *target_size = 2;
8980#endif
8981 break;
8982 }
8983
8984 return insn - insn_buf;
8985}
8986
8987static u32 tc_cls_act_convert_ctx_access(enum bpf_access_type type,
8988 const struct bpf_insn *si,
8989 struct bpf_insn *insn_buf,
8990 struct bpf_prog *prog, u32 *target_size)
8991{
8992 struct bpf_insn *insn = insn_buf;
8993
8994 switch (si->off) {
8995 case offsetof(struct __sk_buff, ifindex):
8996 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
8997 si->dst_reg, si->src_reg,
8998 offsetof(struct sk_buff, dev));
8999 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9000 bpf_target_off(struct net_device, ifindex, 4,
9001 target_size));
9002 break;
9003 default:
9004 return bpf_convert_ctx_access(type, si, insn_buf, prog,
9005 target_size);
9006 }
9007
9008 return insn - insn_buf;
9009}
9010
9011static u32 xdp_convert_ctx_access(enum bpf_access_type type,
9012 const struct bpf_insn *si,
9013 struct bpf_insn *insn_buf,
9014 struct bpf_prog *prog, u32 *target_size)
9015{
9016 struct bpf_insn *insn = insn_buf;
9017
9018 switch (si->off) {
9019 case offsetof(struct xdp_md, data):
9020 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data),
9021 si->dst_reg, si->src_reg,
9022 offsetof(struct xdp_buff, data));
9023 break;
9024 case offsetof(struct xdp_md, data_meta):
9025 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data_meta),
9026 si->dst_reg, si->src_reg,
9027 offsetof(struct xdp_buff, data_meta));
9028 break;
9029 case offsetof(struct xdp_md, data_end):
9030 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data_end),
9031 si->dst_reg, si->src_reg,
9032 offsetof(struct xdp_buff, data_end));
9033 break;
9034 case offsetof(struct xdp_md, ingress_ifindex):
9035 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, rxq),
9036 si->dst_reg, si->src_reg,
9037 offsetof(struct xdp_buff, rxq));
9038 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_rxq_info, dev),
9039 si->dst_reg, si->dst_reg,
9040 offsetof(struct xdp_rxq_info, dev));
9041 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9042 offsetof(struct net_device, ifindex));
9043 break;
9044 case offsetof(struct xdp_md, rx_queue_index):
9045 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, rxq),
9046 si->dst_reg, si->src_reg,
9047 offsetof(struct xdp_buff, rxq));
9048 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9049 offsetof(struct xdp_rxq_info,
9050 queue_index));
9051 break;
9052 case offsetof(struct xdp_md, egress_ifindex):
9053 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, txq),
9054 si->dst_reg, si->src_reg,
9055 offsetof(struct xdp_buff, txq));
9056 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_txq_info, dev),
9057 si->dst_reg, si->dst_reg,
9058 offsetof(struct xdp_txq_info, dev));
9059 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9060 offsetof(struct net_device, ifindex));
9061 break;
9062 }
9063
9064 return insn - insn_buf;
9065}
9066
9067
9068
9069
9070
9071
9072
9073
9074
9075
9076
9077#define SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF) \
9078 do { \
9079 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), si->dst_reg, \
9080 si->src_reg, offsetof(S, F)); \
9081 *insn++ = BPF_LDX_MEM( \
9082 SIZE, si->dst_reg, si->dst_reg, \
9083 bpf_target_off(NS, NF, sizeof_field(NS, NF), \
9084 target_size) \
9085 + OFF); \
9086 } while (0)
9087
9088#define SOCK_ADDR_LOAD_NESTED_FIELD(S, NS, F, NF) \
9089 SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, \
9090 BPF_FIELD_SIZEOF(NS, NF), 0)
9091
9092
9093
9094
9095
9096
9097
9098
9099
9100
9101
9102#define SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE, OFF, TF) \
9103 do { \
9104 int tmp_reg = BPF_REG_9; \
9105 if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg) \
9106 --tmp_reg; \
9107 if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg) \
9108 --tmp_reg; \
9109 *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, tmp_reg, \
9110 offsetof(S, TF)); \
9111 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), tmp_reg, \
9112 si->dst_reg, offsetof(S, F)); \
9113 *insn++ = BPF_STX_MEM(SIZE, tmp_reg, si->src_reg, \
9114 bpf_target_off(NS, NF, sizeof_field(NS, NF), \
9115 target_size) \
9116 + OFF); \
9117 *insn++ = BPF_LDX_MEM(BPF_DW, tmp_reg, si->dst_reg, \
9118 offsetof(S, TF)); \
9119 } while (0)
9120
9121#define SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF, \
9122 TF) \
9123 do { \
9124 if (type == BPF_WRITE) { \
9125 SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE, \
9126 OFF, TF); \
9127 } else { \
9128 SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF( \
9129 S, NS, F, NF, SIZE, OFF); \
9130 } \
9131 } while (0)
9132
9133#define SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD(S, NS, F, NF, TF) \
9134 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF( \
9135 S, NS, F, NF, BPF_FIELD_SIZEOF(NS, NF), 0, TF)
9136
9137static u32 sock_addr_convert_ctx_access(enum bpf_access_type type,
9138 const struct bpf_insn *si,
9139 struct bpf_insn *insn_buf,
9140 struct bpf_prog *prog, u32 *target_size)
9141{
9142 int off, port_size = sizeof_field(struct sockaddr_in6, sin6_port);
9143 struct bpf_insn *insn = insn_buf;
9144
9145 switch (si->off) {
9146 case offsetof(struct bpf_sock_addr, user_family):
9147 SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
9148 struct sockaddr, uaddr, sa_family);
9149 break;
9150
9151 case offsetof(struct bpf_sock_addr, user_ip4):
9152 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
9153 struct bpf_sock_addr_kern, struct sockaddr_in, uaddr,
9154 sin_addr, BPF_SIZE(si->code), 0, tmp_reg);
9155 break;
9156
9157 case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
9158 off = si->off;
9159 off -= offsetof(struct bpf_sock_addr, user_ip6[0]);
9160 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
9161 struct bpf_sock_addr_kern, struct sockaddr_in6, uaddr,
9162 sin6_addr.s6_addr32[0], BPF_SIZE(si->code), off,
9163 tmp_reg);
9164 break;
9165
9166 case offsetof(struct bpf_sock_addr, user_port):
9167
9168
9169
9170
9171
9172
9173
9174 BUILD_BUG_ON(offsetof(struct sockaddr_in, sin_port) !=
9175 offsetof(struct sockaddr_in6, sin6_port));
9176 BUILD_BUG_ON(sizeof_field(struct sockaddr_in, sin_port) !=
9177 sizeof_field(struct sockaddr_in6, sin6_port));
9178
9179 port_size = min(port_size, BPF_LDST_BYTES(si));
9180 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
9181 struct bpf_sock_addr_kern, struct sockaddr_in6, uaddr,
9182 sin6_port, bytes_to_bpf_size(port_size), 0, tmp_reg);
9183 break;
9184
9185 case offsetof(struct bpf_sock_addr, family):
9186 SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
9187 struct sock, sk, sk_family);
9188 break;
9189
9190 case offsetof(struct bpf_sock_addr, type):
9191 SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
9192 struct sock, sk, sk_type);
9193 break;
9194
9195 case offsetof(struct bpf_sock_addr, protocol):
9196 SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
9197 struct sock, sk, sk_protocol);
9198 break;
9199
9200 case offsetof(struct bpf_sock_addr, msg_src_ip4):
9201
9202 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
9203 struct bpf_sock_addr_kern, struct in_addr, t_ctx,
9204 s_addr, BPF_SIZE(si->code), 0, tmp_reg);
9205 break;
9206
9207 case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
9208 msg_src_ip6[3]):
9209 off = si->off;
9210 off -= offsetof(struct bpf_sock_addr, msg_src_ip6[0]);
9211
9212 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
9213 struct bpf_sock_addr_kern, struct in6_addr, t_ctx,
9214 s6_addr32[0], BPF_SIZE(si->code), off, tmp_reg);
9215 break;
9216 case offsetof(struct bpf_sock_addr, sk):
9217 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_addr_kern, sk),
9218 si->dst_reg, si->src_reg,
9219 offsetof(struct bpf_sock_addr_kern, sk));
9220 break;
9221 }
9222
9223 return insn - insn_buf;
9224}
9225
9226static u32 sock_ops_convert_ctx_access(enum bpf_access_type type,
9227 const struct bpf_insn *si,
9228 struct bpf_insn *insn_buf,
9229 struct bpf_prog *prog,
9230 u32 *target_size)
9231{
9232 struct bpf_insn *insn = insn_buf;
9233 int off;
9234
9235
9236#define SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ) \
9237 do { \
9238 int fullsock_reg = si->dst_reg, reg = BPF_REG_9, jmp = 2; \
9239 BUILD_BUG_ON(sizeof_field(OBJ, OBJ_FIELD) > \
9240 sizeof_field(struct bpf_sock_ops, BPF_FIELD)); \
9241 if (si->dst_reg == reg || si->src_reg == reg) \
9242 reg--; \
9243 if (si->dst_reg == reg || si->src_reg == reg) \
9244 reg--; \
9245 if (si->dst_reg == si->src_reg) { \
9246 *insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg, \
9247 offsetof(struct bpf_sock_ops_kern, \
9248 temp)); \
9249 fullsock_reg = reg; \
9250 jmp += 2; \
9251 } \
9252 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
9253 struct bpf_sock_ops_kern, \
9254 is_fullsock), \
9255 fullsock_reg, si->src_reg, \
9256 offsetof(struct bpf_sock_ops_kern, \
9257 is_fullsock)); \
9258 *insn++ = BPF_JMP_IMM(BPF_JEQ, fullsock_reg, 0, jmp); \
9259 if (si->dst_reg == si->src_reg) \
9260 *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \
9261 offsetof(struct bpf_sock_ops_kern, \
9262 temp)); \
9263 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
9264 struct bpf_sock_ops_kern, sk),\
9265 si->dst_reg, si->src_reg, \
9266 offsetof(struct bpf_sock_ops_kern, sk));\
9267 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(OBJ, \
9268 OBJ_FIELD), \
9269 si->dst_reg, si->dst_reg, \
9270 offsetof(OBJ, OBJ_FIELD)); \
9271 if (si->dst_reg == si->src_reg) { \
9272 *insn++ = BPF_JMP_A(1); \
9273 *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \
9274 offsetof(struct bpf_sock_ops_kern, \
9275 temp)); \
9276 } \
9277 } while (0)
9278
9279#define SOCK_OPS_GET_SK() \
9280 do { \
9281 int fullsock_reg = si->dst_reg, reg = BPF_REG_9, jmp = 1; \
9282 if (si->dst_reg == reg || si->src_reg == reg) \
9283 reg--; \
9284 if (si->dst_reg == reg || si->src_reg == reg) \
9285 reg--; \
9286 if (si->dst_reg == si->src_reg) { \
9287 *insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg, \
9288 offsetof(struct bpf_sock_ops_kern, \
9289 temp)); \
9290 fullsock_reg = reg; \
9291 jmp += 2; \
9292 } \
9293 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
9294 struct bpf_sock_ops_kern, \
9295 is_fullsock), \
9296 fullsock_reg, si->src_reg, \
9297 offsetof(struct bpf_sock_ops_kern, \
9298 is_fullsock)); \
9299 *insn++ = BPF_JMP_IMM(BPF_JEQ, fullsock_reg, 0, jmp); \
9300 if (si->dst_reg == si->src_reg) \
9301 *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \
9302 offsetof(struct bpf_sock_ops_kern, \
9303 temp)); \
9304 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
9305 struct bpf_sock_ops_kern, sk),\
9306 si->dst_reg, si->src_reg, \
9307 offsetof(struct bpf_sock_ops_kern, sk));\
9308 if (si->dst_reg == si->src_reg) { \
9309 *insn++ = BPF_JMP_A(1); \
9310 *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \
9311 offsetof(struct bpf_sock_ops_kern, \
9312 temp)); \
9313 } \
9314 } while (0)
9315
9316#define SOCK_OPS_GET_TCP_SOCK_FIELD(FIELD) \
9317 SOCK_OPS_GET_FIELD(FIELD, FIELD, struct tcp_sock)
9318
9319
9320
9321
9322
9323
9324
9325
9326
9327
9328#define SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ) \
9329 do { \
9330 int reg = BPF_REG_9; \
9331 BUILD_BUG_ON(sizeof_field(OBJ, OBJ_FIELD) > \
9332 sizeof_field(struct bpf_sock_ops, BPF_FIELD)); \
9333 if (si->dst_reg == reg || si->src_reg == reg) \
9334 reg--; \
9335 if (si->dst_reg == reg || si->src_reg == reg) \
9336 reg--; \
9337 *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, reg, \
9338 offsetof(struct bpf_sock_ops_kern, \
9339 temp)); \
9340 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
9341 struct bpf_sock_ops_kern, \
9342 is_fullsock), \
9343 reg, si->dst_reg, \
9344 offsetof(struct bpf_sock_ops_kern, \
9345 is_fullsock)); \
9346 *insn++ = BPF_JMP_IMM(BPF_JEQ, reg, 0, 2); \
9347 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
9348 struct bpf_sock_ops_kern, sk),\
9349 reg, si->dst_reg, \
9350 offsetof(struct bpf_sock_ops_kern, sk));\
9351 *insn++ = BPF_STX_MEM(BPF_FIELD_SIZEOF(OBJ, OBJ_FIELD), \
9352 reg, si->src_reg, \
9353 offsetof(OBJ, OBJ_FIELD)); \
9354 *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->dst_reg, \
9355 offsetof(struct bpf_sock_ops_kern, \
9356 temp)); \
9357 } while (0)
9358
9359#define SOCK_OPS_GET_OR_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ, TYPE) \
9360 do { \
9361 if (TYPE == BPF_WRITE) \
9362 SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ); \
9363 else \
9364 SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ); \
9365 } while (0)
9366
9367 if (insn > insn_buf)
9368 return insn - insn_buf;
9369
9370 switch (si->off) {
9371 case offsetof(struct bpf_sock_ops, op):
9372 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
9373 op),
9374 si->dst_reg, si->src_reg,
9375 offsetof(struct bpf_sock_ops_kern, op));
9376 break;
9377
9378 case offsetof(struct bpf_sock_ops, replylong[0]) ...
9379 offsetof(struct bpf_sock_ops, replylong[3]):
9380 BUILD_BUG_ON(sizeof_field(struct bpf_sock_ops, reply) !=
9381 sizeof_field(struct bpf_sock_ops_kern, reply));
9382 BUILD_BUG_ON(sizeof_field(struct bpf_sock_ops, replylong) !=
9383 sizeof_field(struct bpf_sock_ops_kern, replylong));
9384 off = si->off;
9385 off -= offsetof(struct bpf_sock_ops, replylong[0]);
9386 off += offsetof(struct bpf_sock_ops_kern, replylong[0]);
9387 if (type == BPF_WRITE)
9388 *insn++ = BPF_STX_MEM(BPF_W, si->dst_reg, si->src_reg,
9389 off);
9390 else
9391 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
9392 off);
9393 break;
9394
9395 case offsetof(struct bpf_sock_ops, family):
9396 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);
9397
9398 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9399 struct bpf_sock_ops_kern, sk),
9400 si->dst_reg, si->src_reg,
9401 offsetof(struct bpf_sock_ops_kern, sk));
9402 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
9403 offsetof(struct sock_common, skc_family));
9404 break;
9405
9406 case offsetof(struct bpf_sock_ops, remote_ip4):
9407 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);
9408
9409 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9410 struct bpf_sock_ops_kern, sk),
9411 si->dst_reg, si->src_reg,
9412 offsetof(struct bpf_sock_ops_kern, sk));
9413 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9414 offsetof(struct sock_common, skc_daddr));
9415 break;
9416
9417 case offsetof(struct bpf_sock_ops, local_ip4):
9418 BUILD_BUG_ON(sizeof_field(struct sock_common,
9419 skc_rcv_saddr) != 4);
9420
9421 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9422 struct bpf_sock_ops_kern, sk),
9423 si->dst_reg, si->src_reg,
9424 offsetof(struct bpf_sock_ops_kern, sk));
9425 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9426 offsetof(struct sock_common,
9427 skc_rcv_saddr));
9428 break;
9429
9430 case offsetof(struct bpf_sock_ops, remote_ip6[0]) ...
9431 offsetof(struct bpf_sock_ops, remote_ip6[3]):
9432#if IS_ENABLED(CONFIG_IPV6)
9433 BUILD_BUG_ON(sizeof_field(struct sock_common,
9434 skc_v6_daddr.s6_addr32[0]) != 4);
9435
9436 off = si->off;
9437 off -= offsetof(struct bpf_sock_ops, remote_ip6[0]);
9438 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9439 struct bpf_sock_ops_kern, sk),
9440 si->dst_reg, si->src_reg,
9441 offsetof(struct bpf_sock_ops_kern, sk));
9442 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9443 offsetof(struct sock_common,
9444 skc_v6_daddr.s6_addr32[0]) +
9445 off);
9446#else
9447 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
9448#endif
9449 break;
9450
9451 case offsetof(struct bpf_sock_ops, local_ip6[0]) ...
9452 offsetof(struct bpf_sock_ops, local_ip6[3]):
9453#if IS_ENABLED(CONFIG_IPV6)
9454 BUILD_BUG_ON(sizeof_field(struct sock_common,
9455 skc_v6_rcv_saddr.s6_addr32[0]) != 4);
9456
9457 off = si->off;
9458 off -= offsetof(struct bpf_sock_ops, local_ip6[0]);
9459 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9460 struct bpf_sock_ops_kern, sk),
9461 si->dst_reg, si->src_reg,
9462 offsetof(struct bpf_sock_ops_kern, sk));
9463 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9464 offsetof(struct sock_common,
9465 skc_v6_rcv_saddr.s6_addr32[0]) +
9466 off);
9467#else
9468 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
9469#endif
9470 break;
9471
9472 case offsetof(struct bpf_sock_ops, remote_port):
9473 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);
9474
9475 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9476 struct bpf_sock_ops_kern, sk),
9477 si->dst_reg, si->src_reg,
9478 offsetof(struct bpf_sock_ops_kern, sk));
9479 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
9480 offsetof(struct sock_common, skc_dport));
9481#ifndef __BIG_ENDIAN_BITFIELD
9482 *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
9483#endif
9484 break;
9485
9486 case offsetof(struct bpf_sock_ops, local_port):
9487 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);
9488
9489 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9490 struct bpf_sock_ops_kern, sk),
9491 si->dst_reg, si->src_reg,
9492 offsetof(struct bpf_sock_ops_kern, sk));
9493 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
9494 offsetof(struct sock_common, skc_num));
9495 break;
9496
9497 case offsetof(struct bpf_sock_ops, is_fullsock):
9498 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9499 struct bpf_sock_ops_kern,
9500 is_fullsock),
9501 si->dst_reg, si->src_reg,
9502 offsetof(struct bpf_sock_ops_kern,
9503 is_fullsock));
9504 break;
9505
9506 case offsetof(struct bpf_sock_ops, state):
9507 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_state) != 1);
9508
9509 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9510 struct bpf_sock_ops_kern, sk),
9511 si->dst_reg, si->src_reg,
9512 offsetof(struct bpf_sock_ops_kern, sk));
9513 *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->dst_reg,
9514 offsetof(struct sock_common, skc_state));
9515 break;
9516
9517 case offsetof(struct bpf_sock_ops, rtt_min):
9518 BUILD_BUG_ON(sizeof_field(struct tcp_sock, rtt_min) !=
9519 sizeof(struct minmax));
9520 BUILD_BUG_ON(sizeof(struct minmax) <
9521 sizeof(struct minmax_sample));
9522
9523 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9524 struct bpf_sock_ops_kern, sk),
9525 si->dst_reg, si->src_reg,
9526 offsetof(struct bpf_sock_ops_kern, sk));
9527 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9528 offsetof(struct tcp_sock, rtt_min) +
9529 sizeof_field(struct minmax_sample, t));
9530 break;
9531
9532 case offsetof(struct bpf_sock_ops, bpf_sock_ops_cb_flags):
9533 SOCK_OPS_GET_FIELD(bpf_sock_ops_cb_flags, bpf_sock_ops_cb_flags,
9534 struct tcp_sock);
9535 break;
9536
9537 case offsetof(struct bpf_sock_ops, sk_txhash):
9538 SOCK_OPS_GET_OR_SET_FIELD(sk_txhash, sk_txhash,
9539 struct sock, type);
9540 break;
9541 case offsetof(struct bpf_sock_ops, snd_cwnd):
9542 SOCK_OPS_GET_TCP_SOCK_FIELD(snd_cwnd);
9543 break;
9544 case offsetof(struct bpf_sock_ops, srtt_us):
9545 SOCK_OPS_GET_TCP_SOCK_FIELD(srtt_us);
9546 break;
9547 case offsetof(struct bpf_sock_ops, snd_ssthresh):
9548 SOCK_OPS_GET_TCP_SOCK_FIELD(snd_ssthresh);
9549 break;
9550 case offsetof(struct bpf_sock_ops, rcv_nxt):
9551 SOCK_OPS_GET_TCP_SOCK_FIELD(rcv_nxt);
9552 break;
9553 case offsetof(struct bpf_sock_ops, snd_nxt):
9554 SOCK_OPS_GET_TCP_SOCK_FIELD(snd_nxt);
9555 break;
9556 case offsetof(struct bpf_sock_ops, snd_una):
9557 SOCK_OPS_GET_TCP_SOCK_FIELD(snd_una);
9558 break;
9559 case offsetof(struct bpf_sock_ops, mss_cache):
9560 SOCK_OPS_GET_TCP_SOCK_FIELD(mss_cache);
9561 break;
9562 case offsetof(struct bpf_sock_ops, ecn_flags):
9563 SOCK_OPS_GET_TCP_SOCK_FIELD(ecn_flags);
9564 break;
9565 case offsetof(struct bpf_sock_ops, rate_delivered):
9566 SOCK_OPS_GET_TCP_SOCK_FIELD(rate_delivered);
9567 break;
9568 case offsetof(struct bpf_sock_ops, rate_interval_us):
9569 SOCK_OPS_GET_TCP_SOCK_FIELD(rate_interval_us);
9570 break;
9571 case offsetof(struct bpf_sock_ops, packets_out):
9572 SOCK_OPS_GET_TCP_SOCK_FIELD(packets_out);
9573 break;
9574 case offsetof(struct bpf_sock_ops, retrans_out):
9575 SOCK_OPS_GET_TCP_SOCK_FIELD(retrans_out);
9576 break;
9577 case offsetof(struct bpf_sock_ops, total_retrans):
9578 SOCK_OPS_GET_TCP_SOCK_FIELD(total_retrans);
9579 break;
9580 case offsetof(struct bpf_sock_ops, segs_in):
9581 SOCK_OPS_GET_TCP_SOCK_FIELD(segs_in);
9582 break;
9583 case offsetof(struct bpf_sock_ops, data_segs_in):
9584 SOCK_OPS_GET_TCP_SOCK_FIELD(data_segs_in);
9585 break;
9586 case offsetof(struct bpf_sock_ops, segs_out):
9587 SOCK_OPS_GET_TCP_SOCK_FIELD(segs_out);
9588 break;
9589 case offsetof(struct bpf_sock_ops, data_segs_out):
9590 SOCK_OPS_GET_TCP_SOCK_FIELD(data_segs_out);
9591 break;
9592 case offsetof(struct bpf_sock_ops, lost_out):
9593 SOCK_OPS_GET_TCP_SOCK_FIELD(lost_out);
9594 break;
9595 case offsetof(struct bpf_sock_ops, sacked_out):
9596 SOCK_OPS_GET_TCP_SOCK_FIELD(sacked_out);
9597 break;
9598 case offsetof(struct bpf_sock_ops, bytes_received):
9599 SOCK_OPS_GET_TCP_SOCK_FIELD(bytes_received);
9600 break;
9601 case offsetof(struct bpf_sock_ops, bytes_acked):
9602 SOCK_OPS_GET_TCP_SOCK_FIELD(bytes_acked);
9603 break;
9604 case offsetof(struct bpf_sock_ops, sk):
9605 SOCK_OPS_GET_SK();
9606 break;
9607 case offsetof(struct bpf_sock_ops, skb_data_end):
9608 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
9609 skb_data_end),
9610 si->dst_reg, si->src_reg,
9611 offsetof(struct bpf_sock_ops_kern,
9612 skb_data_end));
9613 break;
9614 case offsetof(struct bpf_sock_ops, skb_data):
9615 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
9616 skb),
9617 si->dst_reg, si->src_reg,
9618 offsetof(struct bpf_sock_ops_kern,
9619 skb));
9620 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
9621 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
9622 si->dst_reg, si->dst_reg,
9623 offsetof(struct sk_buff, data));
9624 break;
9625 case offsetof(struct bpf_sock_ops, skb_len):
9626 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
9627 skb),
9628 si->dst_reg, si->src_reg,
9629 offsetof(struct bpf_sock_ops_kern,
9630 skb));
9631 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
9632 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, len),
9633 si->dst_reg, si->dst_reg,
9634 offsetof(struct sk_buff, len));
9635 break;
9636 case offsetof(struct bpf_sock_ops, skb_tcp_flags):
9637 off = offsetof(struct sk_buff, cb);
9638 off += offsetof(struct tcp_skb_cb, tcp_flags);
9639 *target_size = sizeof_field(struct tcp_skb_cb, tcp_flags);
9640 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
9641 skb),
9642 si->dst_reg, si->src_reg,
9643 offsetof(struct bpf_sock_ops_kern,
9644 skb));
9645 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
9646 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct tcp_skb_cb,
9647 tcp_flags),
9648 si->dst_reg, si->dst_reg, off);
9649 break;
9650 }
9651 return insn - insn_buf;
9652}
9653
9654
9655static struct bpf_insn *bpf_convert_data_end_access(const struct bpf_insn *si,
9656 struct bpf_insn *insn)
9657{
9658
9659 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
9660 si->dst_reg, si->src_reg,
9661 offsetof(struct sk_buff, data));
9662
9663 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, len),
9664 BPF_REG_AX, si->src_reg,
9665 offsetof(struct sk_buff, len));
9666
9667 *insn++ = BPF_ALU64_REG(BPF_ADD, si->dst_reg, BPF_REG_AX);
9668
9669 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data_len),
9670 BPF_REG_AX, si->src_reg,
9671 offsetof(struct sk_buff, data_len));
9672
9673 *insn++ = BPF_ALU64_REG(BPF_SUB, si->dst_reg, BPF_REG_AX);
9674
9675 return insn;
9676}
9677
9678static u32 sk_skb_convert_ctx_access(enum bpf_access_type type,
9679 const struct bpf_insn *si,
9680 struct bpf_insn *insn_buf,
9681 struct bpf_prog *prog, u32 *target_size)
9682{
9683 struct bpf_insn *insn = insn_buf;
9684
9685 switch (si->off) {
9686 case offsetof(struct __sk_buff, data_end):
9687 insn = bpf_convert_data_end_access(si, insn);
9688 break;
9689 default:
9690 return bpf_convert_ctx_access(type, si, insn_buf, prog,
9691 target_size);
9692 }
9693
9694 return insn - insn_buf;
9695}
9696
9697static u32 sk_msg_convert_ctx_access(enum bpf_access_type type,
9698 const struct bpf_insn *si,
9699 struct bpf_insn *insn_buf,
9700 struct bpf_prog *prog, u32 *target_size)
9701{
9702 struct bpf_insn *insn = insn_buf;
9703#if IS_ENABLED(CONFIG_IPV6)
9704 int off;
9705#endif
9706
9707
9708 BUILD_BUG_ON(offsetof(struct sk_msg, sg) != 0);
9709
9710 switch (si->off) {
9711 case offsetof(struct sk_msg_md, data):
9712 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, data),
9713 si->dst_reg, si->src_reg,
9714 offsetof(struct sk_msg, data));
9715 break;
9716 case offsetof(struct sk_msg_md, data_end):
9717 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, data_end),
9718 si->dst_reg, si->src_reg,
9719 offsetof(struct sk_msg, data_end));
9720 break;
9721 case offsetof(struct sk_msg_md, family):
9722 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);
9723
9724 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9725 struct sk_msg, sk),
9726 si->dst_reg, si->src_reg,
9727 offsetof(struct sk_msg, sk));
9728 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
9729 offsetof(struct sock_common, skc_family));
9730 break;
9731
9732 case offsetof(struct sk_msg_md, remote_ip4):
9733 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);
9734
9735 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9736 struct sk_msg, sk),
9737 si->dst_reg, si->src_reg,
9738 offsetof(struct sk_msg, sk));
9739 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9740 offsetof(struct sock_common, skc_daddr));
9741 break;
9742
9743 case offsetof(struct sk_msg_md, local_ip4):
9744 BUILD_BUG_ON(sizeof_field(struct sock_common,
9745 skc_rcv_saddr) != 4);
9746
9747 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9748 struct sk_msg, sk),
9749 si->dst_reg, si->src_reg,
9750 offsetof(struct sk_msg, sk));
9751 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9752 offsetof(struct sock_common,
9753 skc_rcv_saddr));
9754 break;
9755
9756 case offsetof(struct sk_msg_md, remote_ip6[0]) ...
9757 offsetof(struct sk_msg_md, remote_ip6[3]):
9758#if IS_ENABLED(CONFIG_IPV6)
9759 BUILD_BUG_ON(sizeof_field(struct sock_common,
9760 skc_v6_daddr.s6_addr32[0]) != 4);
9761
9762 off = si->off;
9763 off -= offsetof(struct sk_msg_md, remote_ip6[0]);
9764 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9765 struct sk_msg, sk),
9766 si->dst_reg, si->src_reg,
9767 offsetof(struct sk_msg, sk));
9768 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9769 offsetof(struct sock_common,
9770 skc_v6_daddr.s6_addr32[0]) +
9771 off);
9772#else
9773 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
9774#endif
9775 break;
9776
9777 case offsetof(struct sk_msg_md, local_ip6[0]) ...
9778 offsetof(struct sk_msg_md, local_ip6[3]):
9779#if IS_ENABLED(CONFIG_IPV6)
9780 BUILD_BUG_ON(sizeof_field(struct sock_common,
9781 skc_v6_rcv_saddr.s6_addr32[0]) != 4);
9782
9783 off = si->off;
9784 off -= offsetof(struct sk_msg_md, local_ip6[0]);
9785 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9786 struct sk_msg, sk),
9787 si->dst_reg, si->src_reg,
9788 offsetof(struct sk_msg, sk));
9789 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9790 offsetof(struct sock_common,
9791 skc_v6_rcv_saddr.s6_addr32[0]) +
9792 off);
9793#else
9794 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
9795#endif
9796 break;
9797
9798 case offsetof(struct sk_msg_md, remote_port):
9799 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);
9800
9801 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9802 struct sk_msg, sk),
9803 si->dst_reg, si->src_reg,
9804 offsetof(struct sk_msg, sk));
9805 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
9806 offsetof(struct sock_common, skc_dport));
9807#ifndef __BIG_ENDIAN_BITFIELD
9808 *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
9809#endif
9810 break;
9811
9812 case offsetof(struct sk_msg_md, local_port):
9813 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);
9814
9815 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
9816 struct sk_msg, sk),
9817 si->dst_reg, si->src_reg,
9818 offsetof(struct sk_msg, sk));
9819 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
9820 offsetof(struct sock_common, skc_num));
9821 break;
9822
9823 case offsetof(struct sk_msg_md, size):
9824 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg_sg, size),
9825 si->dst_reg, si->src_reg,
9826 offsetof(struct sk_msg_sg, size));
9827 break;
9828
9829 case offsetof(struct sk_msg_md, sk):
9830 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, sk),
9831 si->dst_reg, si->src_reg,
9832 offsetof(struct sk_msg, sk));
9833 break;
9834 }
9835
9836 return insn - insn_buf;
9837}
9838
9839const struct bpf_verifier_ops sk_filter_verifier_ops = {
9840 .get_func_proto = sk_filter_func_proto,
9841 .is_valid_access = sk_filter_is_valid_access,
9842 .convert_ctx_access = bpf_convert_ctx_access,
9843 .gen_ld_abs = bpf_gen_ld_abs,
9844};
9845
9846const struct bpf_prog_ops sk_filter_prog_ops = {
9847 .test_run = bpf_prog_test_run_skb,
9848};
9849
9850const struct bpf_verifier_ops tc_cls_act_verifier_ops = {
9851 .get_func_proto = tc_cls_act_func_proto,
9852 .is_valid_access = tc_cls_act_is_valid_access,
9853 .convert_ctx_access = tc_cls_act_convert_ctx_access,
9854 .gen_prologue = tc_cls_act_prologue,
9855 .gen_ld_abs = bpf_gen_ld_abs,
9856 .check_kfunc_call = bpf_prog_test_check_kfunc_call,
9857};
9858
9859const struct bpf_prog_ops tc_cls_act_prog_ops = {
9860 .test_run = bpf_prog_test_run_skb,
9861};
9862
9863const struct bpf_verifier_ops xdp_verifier_ops = {
9864 .get_func_proto = xdp_func_proto,
9865 .is_valid_access = xdp_is_valid_access,
9866 .convert_ctx_access = xdp_convert_ctx_access,
9867 .gen_prologue = bpf_noop_prologue,
9868};
9869
9870const struct bpf_prog_ops xdp_prog_ops = {
9871 .test_run = bpf_prog_test_run_xdp,
9872};
9873
9874const struct bpf_verifier_ops cg_skb_verifier_ops = {
9875 .get_func_proto = cg_skb_func_proto,
9876 .is_valid_access = cg_skb_is_valid_access,
9877 .convert_ctx_access = bpf_convert_ctx_access,
9878};
9879
9880const struct bpf_prog_ops cg_skb_prog_ops = {
9881 .test_run = bpf_prog_test_run_skb,
9882};
9883
9884const struct bpf_verifier_ops lwt_in_verifier_ops = {
9885 .get_func_proto = lwt_in_func_proto,
9886 .is_valid_access = lwt_is_valid_access,
9887 .convert_ctx_access = bpf_convert_ctx_access,
9888};
9889
9890const struct bpf_prog_ops lwt_in_prog_ops = {
9891 .test_run = bpf_prog_test_run_skb,
9892};
9893
9894const struct bpf_verifier_ops lwt_out_verifier_ops = {
9895 .get_func_proto = lwt_out_func_proto,
9896 .is_valid_access = lwt_is_valid_access,
9897 .convert_ctx_access = bpf_convert_ctx_access,
9898};
9899
9900const struct bpf_prog_ops lwt_out_prog_ops = {
9901 .test_run = bpf_prog_test_run_skb,
9902};
9903
9904const struct bpf_verifier_ops lwt_xmit_verifier_ops = {
9905 .get_func_proto = lwt_xmit_func_proto,
9906 .is_valid_access = lwt_is_valid_access,
9907 .convert_ctx_access = bpf_convert_ctx_access,
9908 .gen_prologue = tc_cls_act_prologue,
9909};
9910
9911const struct bpf_prog_ops lwt_xmit_prog_ops = {
9912 .test_run = bpf_prog_test_run_skb,
9913};
9914
9915const struct bpf_verifier_ops lwt_seg6local_verifier_ops = {
9916 .get_func_proto = lwt_seg6local_func_proto,
9917 .is_valid_access = lwt_is_valid_access,
9918 .convert_ctx_access = bpf_convert_ctx_access,
9919};
9920
9921const struct bpf_prog_ops lwt_seg6local_prog_ops = {
9922 .test_run = bpf_prog_test_run_skb,
9923};
9924
9925const struct bpf_verifier_ops cg_sock_verifier_ops = {
9926 .get_func_proto = sock_filter_func_proto,
9927 .is_valid_access = sock_filter_is_valid_access,
9928 .convert_ctx_access = bpf_sock_convert_ctx_access,
9929};
9930
9931const struct bpf_prog_ops cg_sock_prog_ops = {
9932};
9933
9934const struct bpf_verifier_ops cg_sock_addr_verifier_ops = {
9935 .get_func_proto = sock_addr_func_proto,
9936 .is_valid_access = sock_addr_is_valid_access,
9937 .convert_ctx_access = sock_addr_convert_ctx_access,
9938};
9939
9940const struct bpf_prog_ops cg_sock_addr_prog_ops = {
9941};
9942
9943const struct bpf_verifier_ops sock_ops_verifier_ops = {
9944 .get_func_proto = sock_ops_func_proto,
9945 .is_valid_access = sock_ops_is_valid_access,
9946 .convert_ctx_access = sock_ops_convert_ctx_access,
9947};
9948
9949const struct bpf_prog_ops sock_ops_prog_ops = {
9950};
9951
9952const struct bpf_verifier_ops sk_skb_verifier_ops = {
9953 .get_func_proto = sk_skb_func_proto,
9954 .is_valid_access = sk_skb_is_valid_access,
9955 .convert_ctx_access = sk_skb_convert_ctx_access,
9956 .gen_prologue = sk_skb_prologue,
9957};
9958
9959const struct bpf_prog_ops sk_skb_prog_ops = {
9960};
9961
9962const struct bpf_verifier_ops sk_msg_verifier_ops = {
9963 .get_func_proto = sk_msg_func_proto,
9964 .is_valid_access = sk_msg_is_valid_access,
9965 .convert_ctx_access = sk_msg_convert_ctx_access,
9966 .gen_prologue = bpf_noop_prologue,
9967};
9968
9969const struct bpf_prog_ops sk_msg_prog_ops = {
9970};
9971
9972const struct bpf_verifier_ops flow_dissector_verifier_ops = {
9973 .get_func_proto = flow_dissector_func_proto,
9974 .is_valid_access = flow_dissector_is_valid_access,
9975 .convert_ctx_access = flow_dissector_convert_ctx_access,
9976};
9977
9978const struct bpf_prog_ops flow_dissector_prog_ops = {
9979 .test_run = bpf_prog_test_run_flow_dissector,
9980};
9981
9982int sk_detach_filter(struct sock *sk)
9983{
9984 int ret = -ENOENT;
9985 struct sk_filter *filter;
9986
9987 if (sock_flag(sk, SOCK_FILTER_LOCKED))
9988 return -EPERM;
9989
9990 filter = rcu_dereference_protected(sk->sk_filter,
9991 lockdep_sock_is_held(sk));
9992 if (filter) {
9993 RCU_INIT_POINTER(sk->sk_filter, NULL);
9994 sk_filter_uncharge(sk, filter);
9995 ret = 0;
9996 }
9997
9998 return ret;
9999}
10000EXPORT_SYMBOL_GPL(sk_detach_filter);
10001
10002int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf,
10003 unsigned int len)
10004{
10005 struct sock_fprog_kern *fprog;
10006 struct sk_filter *filter;
10007 int ret = 0;
10008
10009 lock_sock(sk);
10010 filter = rcu_dereference_protected(sk->sk_filter,
10011 lockdep_sock_is_held(sk));
10012 if (!filter)
10013 goto out;
10014
10015
10016
10017
10018
10019 ret = -EACCES;
10020 fprog = filter->prog->orig_prog;
10021 if (!fprog)
10022 goto out;
10023
10024 ret = fprog->len;
10025 if (!len)
10026
10027 goto out;
10028
10029 ret = -EINVAL;
10030 if (len < fprog->len)
10031 goto out;
10032
10033 ret = -EFAULT;
10034 if (copy_to_user(ubuf, fprog->filter, bpf_classic_proglen(fprog)))
10035 goto out;
10036
10037
10038
10039
10040 ret = fprog->len;
10041out:
10042 release_sock(sk);
10043 return ret;
10044}
10045
10046#ifdef CONFIG_INET
10047static void bpf_init_reuseport_kern(struct sk_reuseport_kern *reuse_kern,
10048 struct sock_reuseport *reuse,
10049 struct sock *sk, struct sk_buff *skb,
10050 struct sock *migrating_sk,
10051 u32 hash)
10052{
10053 reuse_kern->skb = skb;
10054 reuse_kern->sk = sk;
10055 reuse_kern->selected_sk = NULL;
10056 reuse_kern->migrating_sk = migrating_sk;
10057 reuse_kern->data_end = skb->data + skb_headlen(skb);
10058 reuse_kern->hash = hash;
10059 reuse_kern->reuseport_id = reuse->reuseport_id;
10060 reuse_kern->bind_inany = reuse->bind_inany;
10061}
10062
10063struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
10064 struct bpf_prog *prog, struct sk_buff *skb,
10065 struct sock *migrating_sk,
10066 u32 hash)
10067{
10068 struct sk_reuseport_kern reuse_kern;
10069 enum sk_action action;
10070
10071 bpf_init_reuseport_kern(&reuse_kern, reuse, sk, skb, migrating_sk, hash);
10072 action = BPF_PROG_RUN(prog, &reuse_kern);
10073
10074 if (action == SK_PASS)
10075 return reuse_kern.selected_sk;
10076 else
10077 return ERR_PTR(-ECONNREFUSED);
10078}
10079
10080BPF_CALL_4(sk_select_reuseport, struct sk_reuseport_kern *, reuse_kern,
10081 struct bpf_map *, map, void *, key, u32, flags)
10082{
10083 bool is_sockarray = map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY;
10084 struct sock_reuseport *reuse;
10085 struct sock *selected_sk;
10086
10087 selected_sk = map->ops->map_lookup_elem(map, key);
10088 if (!selected_sk)
10089 return -ENOENT;
10090
10091 reuse = rcu_dereference(selected_sk->sk_reuseport_cb);
10092 if (!reuse) {
10093
10094 if (sk_is_refcounted(selected_sk))
10095 sock_put(selected_sk);
10096
10097
10098
10099
10100
10101
10102
10103
10104 return is_sockarray ? -ENOENT : -EINVAL;
10105 }
10106
10107 if (unlikely(reuse->reuseport_id != reuse_kern->reuseport_id)) {
10108 struct sock *sk = reuse_kern->sk;
10109
10110 if (sk->sk_protocol != selected_sk->sk_protocol)
10111 return -EPROTOTYPE;
10112 else if (sk->sk_family != selected_sk->sk_family)
10113 return -EAFNOSUPPORT;
10114
10115
10116 return -EBADFD;
10117 }
10118
10119 reuse_kern->selected_sk = selected_sk;
10120
10121 return 0;
10122}
10123
10124static const struct bpf_func_proto sk_select_reuseport_proto = {
10125 .func = sk_select_reuseport,
10126 .gpl_only = false,
10127 .ret_type = RET_INTEGER,
10128 .arg1_type = ARG_PTR_TO_CTX,
10129 .arg2_type = ARG_CONST_MAP_PTR,
10130 .arg3_type = ARG_PTR_TO_MAP_KEY,
10131 .arg4_type = ARG_ANYTHING,
10132};
10133
10134BPF_CALL_4(sk_reuseport_load_bytes,
10135 const struct sk_reuseport_kern *, reuse_kern, u32, offset,
10136 void *, to, u32, len)
10137{
10138 return ____bpf_skb_load_bytes(reuse_kern->skb, offset, to, len);
10139}
10140
10141static const struct bpf_func_proto sk_reuseport_load_bytes_proto = {
10142 .func = sk_reuseport_load_bytes,
10143 .gpl_only = false,
10144 .ret_type = RET_INTEGER,
10145 .arg1_type = ARG_PTR_TO_CTX,
10146 .arg2_type = ARG_ANYTHING,
10147 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
10148 .arg4_type = ARG_CONST_SIZE,
10149};
10150
10151BPF_CALL_5(sk_reuseport_load_bytes_relative,
10152 const struct sk_reuseport_kern *, reuse_kern, u32, offset,
10153 void *, to, u32, len, u32, start_header)
10154{
10155 return ____bpf_skb_load_bytes_relative(reuse_kern->skb, offset, to,
10156 len, start_header);
10157}
10158
10159static const struct bpf_func_proto sk_reuseport_load_bytes_relative_proto = {
10160 .func = sk_reuseport_load_bytes_relative,
10161 .gpl_only = false,
10162 .ret_type = RET_INTEGER,
10163 .arg1_type = ARG_PTR_TO_CTX,
10164 .arg2_type = ARG_ANYTHING,
10165 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
10166 .arg4_type = ARG_CONST_SIZE,
10167 .arg5_type = ARG_ANYTHING,
10168};
10169
10170static const struct bpf_func_proto *
10171sk_reuseport_func_proto(enum bpf_func_id func_id,
10172 const struct bpf_prog *prog)
10173{
10174 switch (func_id) {
10175 case BPF_FUNC_sk_select_reuseport:
10176 return &sk_select_reuseport_proto;
10177 case BPF_FUNC_skb_load_bytes:
10178 return &sk_reuseport_load_bytes_proto;
10179 case BPF_FUNC_skb_load_bytes_relative:
10180 return &sk_reuseport_load_bytes_relative_proto;
10181 case BPF_FUNC_get_socket_cookie:
10182 return &bpf_get_socket_ptr_cookie_proto;
10183 default:
10184 return bpf_base_func_proto(func_id);
10185 }
10186}
10187
10188static bool
10189sk_reuseport_is_valid_access(int off, int size,
10190 enum bpf_access_type type,
10191 const struct bpf_prog *prog,
10192 struct bpf_insn_access_aux *info)
10193{
10194 const u32 size_default = sizeof(__u32);
10195
10196 if (off < 0 || off >= sizeof(struct sk_reuseport_md) ||
10197 off % size || type != BPF_READ)
10198 return false;
10199
10200 switch (off) {
10201 case offsetof(struct sk_reuseport_md, data):
10202 info->reg_type = PTR_TO_PACKET;
10203 return size == sizeof(__u64);
10204
10205 case offsetof(struct sk_reuseport_md, data_end):
10206 info->reg_type = PTR_TO_PACKET_END;
10207 return size == sizeof(__u64);
10208
10209 case offsetof(struct sk_reuseport_md, hash):
10210 return size == size_default;
10211
10212 case offsetof(struct sk_reuseport_md, sk):
10213 info->reg_type = PTR_TO_SOCKET;
10214 return size == sizeof(__u64);
10215
10216 case offsetof(struct sk_reuseport_md, migrating_sk):
10217 info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL;
10218 return size == sizeof(__u64);
10219
10220
10221 case bpf_ctx_range(struct sk_reuseport_md, eth_protocol):
10222 if (size < sizeof_field(struct sk_buff, protocol))
10223 return false;
10224 fallthrough;
10225 case bpf_ctx_range(struct sk_reuseport_md, ip_protocol):
10226 case bpf_ctx_range(struct sk_reuseport_md, bind_inany):
10227 case bpf_ctx_range(struct sk_reuseport_md, len):
10228 bpf_ctx_record_field_size(info, size_default);
10229 return bpf_ctx_narrow_access_ok(off, size, size_default);
10230
10231 default:
10232 return false;
10233 }
10234}
10235
10236#define SK_REUSEPORT_LOAD_FIELD(F) ({ \
10237 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_reuseport_kern, F), \
10238 si->dst_reg, si->src_reg, \
10239 bpf_target_off(struct sk_reuseport_kern, F, \
10240 sizeof_field(struct sk_reuseport_kern, F), \
10241 target_size)); \
10242 })
10243
10244#define SK_REUSEPORT_LOAD_SKB_FIELD(SKB_FIELD) \
10245 SOCK_ADDR_LOAD_NESTED_FIELD(struct sk_reuseport_kern, \
10246 struct sk_buff, \
10247 skb, \
10248 SKB_FIELD)
10249
10250#define SK_REUSEPORT_LOAD_SK_FIELD(SK_FIELD) \
10251 SOCK_ADDR_LOAD_NESTED_FIELD(struct sk_reuseport_kern, \
10252 struct sock, \
10253 sk, \
10254 SK_FIELD)
10255
10256static u32 sk_reuseport_convert_ctx_access(enum bpf_access_type type,
10257 const struct bpf_insn *si,
10258 struct bpf_insn *insn_buf,
10259 struct bpf_prog *prog,
10260 u32 *target_size)
10261{
10262 struct bpf_insn *insn = insn_buf;
10263
10264 switch (si->off) {
10265 case offsetof(struct sk_reuseport_md, data):
10266 SK_REUSEPORT_LOAD_SKB_FIELD(data);
10267 break;
10268
10269 case offsetof(struct sk_reuseport_md, len):
10270 SK_REUSEPORT_LOAD_SKB_FIELD(len);
10271 break;
10272
10273 case offsetof(struct sk_reuseport_md, eth_protocol):
10274 SK_REUSEPORT_LOAD_SKB_FIELD(protocol);
10275 break;
10276
10277 case offsetof(struct sk_reuseport_md, ip_protocol):
10278 SK_REUSEPORT_LOAD_SK_FIELD(sk_protocol);
10279 break;
10280
10281 case offsetof(struct sk_reuseport_md, data_end):
10282 SK_REUSEPORT_LOAD_FIELD(data_end);
10283 break;
10284
10285 case offsetof(struct sk_reuseport_md, hash):
10286 SK_REUSEPORT_LOAD_FIELD(hash);
10287 break;
10288
10289 case offsetof(struct sk_reuseport_md, bind_inany):
10290 SK_REUSEPORT_LOAD_FIELD(bind_inany);
10291 break;
10292
10293 case offsetof(struct sk_reuseport_md, sk):
10294 SK_REUSEPORT_LOAD_FIELD(sk);
10295 break;
10296
10297 case offsetof(struct sk_reuseport_md, migrating_sk):
10298 SK_REUSEPORT_LOAD_FIELD(migrating_sk);
10299 break;
10300 }
10301
10302 return insn - insn_buf;
10303}
10304
10305const struct bpf_verifier_ops sk_reuseport_verifier_ops = {
10306 .get_func_proto = sk_reuseport_func_proto,
10307 .is_valid_access = sk_reuseport_is_valid_access,
10308 .convert_ctx_access = sk_reuseport_convert_ctx_access,
10309};
10310
10311const struct bpf_prog_ops sk_reuseport_prog_ops = {
10312};
10313
10314DEFINE_STATIC_KEY_FALSE(bpf_sk_lookup_enabled);
10315EXPORT_SYMBOL(bpf_sk_lookup_enabled);
10316
10317BPF_CALL_3(bpf_sk_lookup_assign, struct bpf_sk_lookup_kern *, ctx,
10318 struct sock *, sk, u64, flags)
10319{
10320 if (unlikely(flags & ~(BPF_SK_LOOKUP_F_REPLACE |
10321 BPF_SK_LOOKUP_F_NO_REUSEPORT)))
10322 return -EINVAL;
10323 if (unlikely(sk && sk_is_refcounted(sk)))
10324 return -ESOCKTNOSUPPORT;
10325 if (unlikely(sk && sk->sk_state == TCP_ESTABLISHED))
10326 return -ESOCKTNOSUPPORT;
10327
10328
10329 if (sk && sk->sk_protocol != ctx->protocol)
10330 return -EPROTOTYPE;
10331 if (sk && sk->sk_family != ctx->family &&
10332 (sk->sk_family == AF_INET || ipv6_only_sock(sk)))
10333 return -EAFNOSUPPORT;
10334
10335 if (ctx->selected_sk && !(flags & BPF_SK_LOOKUP_F_REPLACE))
10336 return -EEXIST;
10337
10338
10339 ctx->selected_sk = sk;
10340 ctx->no_reuseport = flags & BPF_SK_LOOKUP_F_NO_REUSEPORT;
10341 return 0;
10342}
10343
10344static const struct bpf_func_proto bpf_sk_lookup_assign_proto = {
10345 .func = bpf_sk_lookup_assign,
10346 .gpl_only = false,
10347 .ret_type = RET_INTEGER,
10348 .arg1_type = ARG_PTR_TO_CTX,
10349 .arg2_type = ARG_PTR_TO_SOCKET_OR_NULL,
10350 .arg3_type = ARG_ANYTHING,
10351};
10352
10353static const struct bpf_func_proto *
10354sk_lookup_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
10355{
10356 switch (func_id) {
10357 case BPF_FUNC_perf_event_output:
10358 return &bpf_event_output_data_proto;
10359 case BPF_FUNC_sk_assign:
10360 return &bpf_sk_lookup_assign_proto;
10361 case BPF_FUNC_sk_release:
10362 return &bpf_sk_release_proto;
10363 default:
10364 return bpf_sk_base_func_proto(func_id);
10365 }
10366}
10367
10368static bool sk_lookup_is_valid_access(int off, int size,
10369 enum bpf_access_type type,
10370 const struct bpf_prog *prog,
10371 struct bpf_insn_access_aux *info)
10372{
10373 if (off < 0 || off >= sizeof(struct bpf_sk_lookup))
10374 return false;
10375 if (off % size != 0)
10376 return false;
10377 if (type != BPF_READ)
10378 return false;
10379
10380 switch (off) {
10381 case offsetof(struct bpf_sk_lookup, sk):
10382 info->reg_type = PTR_TO_SOCKET_OR_NULL;
10383 return size == sizeof(__u64);
10384
10385 case bpf_ctx_range(struct bpf_sk_lookup, family):
10386 case bpf_ctx_range(struct bpf_sk_lookup, protocol):
10387 case bpf_ctx_range(struct bpf_sk_lookup, remote_ip4):
10388 case bpf_ctx_range(struct bpf_sk_lookup, local_ip4):
10389 case bpf_ctx_range_till(struct bpf_sk_lookup, remote_ip6[0], remote_ip6[3]):
10390 case bpf_ctx_range_till(struct bpf_sk_lookup, local_ip6[0], local_ip6[3]):
10391 case bpf_ctx_range(struct bpf_sk_lookup, remote_port):
10392 case bpf_ctx_range(struct bpf_sk_lookup, local_port):
10393 bpf_ctx_record_field_size(info, sizeof(__u32));
10394 return bpf_ctx_narrow_access_ok(off, size, sizeof(__u32));
10395
10396 default:
10397 return false;
10398 }
10399}
10400
10401static u32 sk_lookup_convert_ctx_access(enum bpf_access_type type,
10402 const struct bpf_insn *si,
10403 struct bpf_insn *insn_buf,
10404 struct bpf_prog *prog,
10405 u32 *target_size)
10406{
10407 struct bpf_insn *insn = insn_buf;
10408
10409 switch (si->off) {
10410 case offsetof(struct bpf_sk_lookup, sk):
10411 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
10412 offsetof(struct bpf_sk_lookup_kern, selected_sk));
10413 break;
10414
10415 case offsetof(struct bpf_sk_lookup, family):
10416 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
10417 bpf_target_off(struct bpf_sk_lookup_kern,
10418 family, 2, target_size));
10419 break;
10420
10421 case offsetof(struct bpf_sk_lookup, protocol):
10422 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
10423 bpf_target_off(struct bpf_sk_lookup_kern,
10424 protocol, 2, target_size));
10425 break;
10426
10427 case offsetof(struct bpf_sk_lookup, remote_ip4):
10428 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10429 bpf_target_off(struct bpf_sk_lookup_kern,
10430 v4.saddr, 4, target_size));
10431 break;
10432
10433 case offsetof(struct bpf_sk_lookup, local_ip4):
10434 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10435 bpf_target_off(struct bpf_sk_lookup_kern,
10436 v4.daddr, 4, target_size));
10437 break;
10438
10439 case bpf_ctx_range_till(struct bpf_sk_lookup,
10440 remote_ip6[0], remote_ip6[3]): {
10441#if IS_ENABLED(CONFIG_IPV6)
10442 int off = si->off;
10443
10444 off -= offsetof(struct bpf_sk_lookup, remote_ip6[0]);
10445 off += bpf_target_off(struct in6_addr, s6_addr32[0], 4, target_size);
10446 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
10447 offsetof(struct bpf_sk_lookup_kern, v6.saddr));
10448 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
10449 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, off);
10450#else
10451 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
10452#endif
10453 break;
10454 }
10455 case bpf_ctx_range_till(struct bpf_sk_lookup,
10456 local_ip6[0], local_ip6[3]): {
10457#if IS_ENABLED(CONFIG_IPV6)
10458 int off = si->off;
10459
10460 off -= offsetof(struct bpf_sk_lookup, local_ip6[0]);
10461 off += bpf_target_off(struct in6_addr, s6_addr32[0], 4, target_size);
10462 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
10463 offsetof(struct bpf_sk_lookup_kern, v6.daddr));
10464 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
10465 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, off);
10466#else
10467 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
10468#endif
10469 break;
10470 }
10471 case offsetof(struct bpf_sk_lookup, remote_port):
10472 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
10473 bpf_target_off(struct bpf_sk_lookup_kern,
10474 sport, 2, target_size));
10475 break;
10476
10477 case offsetof(struct bpf_sk_lookup, local_port):
10478 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
10479 bpf_target_off(struct bpf_sk_lookup_kern,
10480 dport, 2, target_size));
10481 break;
10482 }
10483
10484 return insn - insn_buf;
10485}
10486
10487const struct bpf_prog_ops sk_lookup_prog_ops = {
10488 .test_run = bpf_prog_test_run_sk_lookup,
10489};
10490
10491const struct bpf_verifier_ops sk_lookup_verifier_ops = {
10492 .get_func_proto = sk_lookup_func_proto,
10493 .is_valid_access = sk_lookup_is_valid_access,
10494 .convert_ctx_access = sk_lookup_convert_ctx_access,
10495};
10496
10497#endif
10498
10499DEFINE_BPF_DISPATCHER(xdp)
10500
10501void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog)
10502{
10503 bpf_dispatcher_change_prog(BPF_DISPATCHER_PTR(xdp), prev_prog, prog);
10504}
10505
10506#ifdef CONFIG_DEBUG_INFO_BTF
10507BTF_ID_LIST_GLOBAL(btf_sock_ids)
10508#define BTF_SOCK_TYPE(name, type) BTF_ID(struct, type)
10509BTF_SOCK_TYPE_xxx
10510#undef BTF_SOCK_TYPE
10511#else
10512u32 btf_sock_ids[MAX_BTF_SOCK_TYPE];
10513#endif
10514
10515BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk)
10516{
10517
10518
10519
10520 BTF_TYPE_EMIT(struct tcp6_sock);
10521 if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP &&
10522 sk->sk_family == AF_INET6)
10523 return (unsigned long)sk;
10524
10525 return (unsigned long)NULL;
10526}
10527
10528const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto = {
10529 .func = bpf_skc_to_tcp6_sock,
10530 .gpl_only = false,
10531 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
10532 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
10533 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP6],
10534};
10535
10536BPF_CALL_1(bpf_skc_to_tcp_sock, struct sock *, sk)
10537{
10538 if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
10539 return (unsigned long)sk;
10540
10541 return (unsigned long)NULL;
10542}
10543
10544const struct bpf_func_proto bpf_skc_to_tcp_sock_proto = {
10545 .func = bpf_skc_to_tcp_sock,
10546 .gpl_only = false,
10547 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
10548 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
10549 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
10550};
10551
10552BPF_CALL_1(bpf_skc_to_tcp_timewait_sock, struct sock *, sk)
10553{
10554
10555
10556
10557 BTF_TYPE_EMIT(struct inet_timewait_sock);
10558 BTF_TYPE_EMIT(struct tcp_timewait_sock);
10559
10560#ifdef CONFIG_INET
10561 if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_TIME_WAIT)
10562 return (unsigned long)sk;
10563#endif
10564
10565#if IS_BUILTIN(CONFIG_IPV6)
10566 if (sk && sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_TIME_WAIT)
10567 return (unsigned long)sk;
10568#endif
10569
10570 return (unsigned long)NULL;
10571}
10572
10573const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto = {
10574 .func = bpf_skc_to_tcp_timewait_sock,
10575 .gpl_only = false,
10576 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
10577 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
10578 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP_TW],
10579};
10580
10581BPF_CALL_1(bpf_skc_to_tcp_request_sock, struct sock *, sk)
10582{
10583#ifdef CONFIG_INET
10584 if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_NEW_SYN_RECV)
10585 return (unsigned long)sk;
10586#endif
10587
10588#if IS_BUILTIN(CONFIG_IPV6)
10589 if (sk && sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_NEW_SYN_RECV)
10590 return (unsigned long)sk;
10591#endif
10592
10593 return (unsigned long)NULL;
10594}
10595
10596const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto = {
10597 .func = bpf_skc_to_tcp_request_sock,
10598 .gpl_only = false,
10599 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
10600 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
10601 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP_REQ],
10602};
10603
10604BPF_CALL_1(bpf_skc_to_udp6_sock, struct sock *, sk)
10605{
10606
10607
10608
10609 BTF_TYPE_EMIT(struct udp6_sock);
10610 if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_UDP &&
10611 sk->sk_type == SOCK_DGRAM && sk->sk_family == AF_INET6)
10612 return (unsigned long)sk;
10613
10614 return (unsigned long)NULL;
10615}
10616
10617const struct bpf_func_proto bpf_skc_to_udp6_sock_proto = {
10618 .func = bpf_skc_to_udp6_sock,
10619 .gpl_only = false,
10620 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
10621 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
10622 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_UDP6],
10623};
10624
10625BPF_CALL_1(bpf_sock_from_file, struct file *, file)
10626{
10627 return (unsigned long)sock_from_file(file);
10628}
10629
10630BTF_ID_LIST(bpf_sock_from_file_btf_ids)
10631BTF_ID(struct, socket)
10632BTF_ID(struct, file)
10633
10634const struct bpf_func_proto bpf_sock_from_file_proto = {
10635 .func = bpf_sock_from_file,
10636 .gpl_only = false,
10637 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
10638 .ret_btf_id = &bpf_sock_from_file_btf_ids[0],
10639 .arg1_type = ARG_PTR_TO_BTF_ID,
10640 .arg1_btf_id = &bpf_sock_from_file_btf_ids[1],
10641};
10642
10643static const struct bpf_func_proto *
10644bpf_sk_base_func_proto(enum bpf_func_id func_id)
10645{
10646 const struct bpf_func_proto *func;
10647
10648 switch (func_id) {
10649 case BPF_FUNC_skc_to_tcp6_sock:
10650 func = &bpf_skc_to_tcp6_sock_proto;
10651 break;
10652 case BPF_FUNC_skc_to_tcp_sock:
10653 func = &bpf_skc_to_tcp_sock_proto;
10654 break;
10655 case BPF_FUNC_skc_to_tcp_timewait_sock:
10656 func = &bpf_skc_to_tcp_timewait_sock_proto;
10657 break;
10658 case BPF_FUNC_skc_to_tcp_request_sock:
10659 func = &bpf_skc_to_tcp_request_sock_proto;
10660 break;
10661 case BPF_FUNC_skc_to_udp6_sock:
10662 func = &bpf_skc_to_udp6_sock_proto;
10663 break;
10664 default:
10665 return bpf_base_func_proto(func_id);
10666 }
10667
10668 if (!perfmon_capable())
10669 return NULL;
10670
10671 return func;
10672}
10673