1
2
3
4#include <linux/bpf.h>
5#include <linux/btf_ids.h>
6#include <linux/slab.h>
7#include <linux/vmalloc.h>
8#include <linux/etherdevice.h>
9#include <linux/filter.h>
10#include <linux/rcupdate_trace.h>
11#include <linux/sched/signal.h>
12#include <net/bpf_sk_storage.h>
13#include <net/sock.h>
14#include <net/tcp.h>
15#include <net/net_namespace.h>
16#include <linux/error-injection.h>
17#include <linux/smp.h>
18#include <linux/sock_diag.h>
19
20#define CREATE_TRACE_POINTS
21#include <trace/events/bpf_test_run.h>
22
23struct bpf_test_timer {
24 enum { NO_PREEMPT, NO_MIGRATE } mode;
25 u32 i;
26 u64 time_start, time_spent;
27};
28
29static void bpf_test_timer_enter(struct bpf_test_timer *t)
30 __acquires(rcu)
31{
32 rcu_read_lock();
33 if (t->mode == NO_PREEMPT)
34 preempt_disable();
35 else
36 migrate_disable();
37
38 t->time_start = ktime_get_ns();
39}
40
41static void bpf_test_timer_leave(struct bpf_test_timer *t)
42 __releases(rcu)
43{
44 t->time_start = 0;
45
46 if (t->mode == NO_PREEMPT)
47 preempt_enable();
48 else
49 migrate_enable();
50 rcu_read_unlock();
51}
52
53static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration)
54 __must_hold(rcu)
55{
56 t->i++;
57 if (t->i >= repeat) {
58
59 t->time_spent += ktime_get_ns() - t->time_start;
60 do_div(t->time_spent, t->i);
61 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
62 *err = 0;
63 goto reset;
64 }
65
66 if (signal_pending(current)) {
67
68 *err = -EINTR;
69 goto reset;
70 }
71
72 if (need_resched()) {
73
74 t->time_spent += ktime_get_ns() - t->time_start;
75 bpf_test_timer_leave(t);
76 cond_resched();
77 bpf_test_timer_enter(t);
78 }
79
80
81 return true;
82
83reset:
84 t->i = 0;
85 return false;
86}
87
88static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
89 u32 *retval, u32 *time, bool xdp)
90{
91 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL };
92 struct bpf_test_timer t = { NO_MIGRATE };
93 enum bpf_cgroup_storage_type stype;
94 int ret;
95
96 for_each_cgroup_storage_type(stype) {
97 storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
98 if (IS_ERR(storage[stype])) {
99 storage[stype] = NULL;
100 for_each_cgroup_storage_type(stype)
101 bpf_cgroup_storage_free(storage[stype]);
102 return -ENOMEM;
103 }
104 }
105
106 if (!repeat)
107 repeat = 1;
108
109 bpf_test_timer_enter(&t);
110 do {
111 ret = bpf_cgroup_storage_set(storage);
112 if (ret)
113 break;
114
115 if (xdp)
116 *retval = bpf_prog_run_xdp(prog, ctx);
117 else
118 *retval = BPF_PROG_RUN(prog, ctx);
119
120 bpf_cgroup_storage_unset();
121 } while (bpf_test_timer_continue(&t, repeat, &ret, time));
122 bpf_test_timer_leave(&t);
123
124 for_each_cgroup_storage_type(stype)
125 bpf_cgroup_storage_free(storage[stype]);
126
127 return ret;
128}
129
130static int bpf_test_finish(const union bpf_attr *kattr,
131 union bpf_attr __user *uattr, const void *data,
132 u32 size, u32 retval, u32 duration)
133{
134 void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
135 int err = -EFAULT;
136 u32 copy_size = size;
137
138
139
140
141 if (kattr->test.data_size_out &&
142 copy_size > kattr->test.data_size_out) {
143 copy_size = kattr->test.data_size_out;
144 err = -ENOSPC;
145 }
146
147 if (data_out && copy_to_user(data_out, data, copy_size))
148 goto out;
149 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
150 goto out;
151 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
152 goto out;
153 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
154 goto out;
155 if (err != -ENOSPC)
156 err = 0;
157out:
158 trace_bpf_test_finish(&err);
159 return err;
160}
161
162
163
164
165
166__diag_push();
167__diag_ignore(GCC, 8, "-Wmissing-prototypes",
168 "Global functions as their definitions will be in vmlinux BTF");
169int noinline bpf_fentry_test1(int a)
170{
171 return a + 1;
172}
173
174int noinline bpf_fentry_test2(int a, u64 b)
175{
176 return a + b;
177}
178
179int noinline bpf_fentry_test3(char a, int b, u64 c)
180{
181 return a + b + c;
182}
183
184int noinline bpf_fentry_test4(void *a, char b, int c, u64 d)
185{
186 return (long)a + b + c + d;
187}
188
189int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e)
190{
191 return a + (long)b + c + d + e;
192}
193
194int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f)
195{
196 return a + (long)b + c + d + (long)e + f;
197}
198
199struct bpf_fentry_test_t {
200 struct bpf_fentry_test_t *a;
201};
202
203int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg)
204{
205 return (long)arg;
206}
207
208int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg)
209{
210 return (long)arg->a;
211}
212
213int noinline bpf_modify_return_test(int a, int *b)
214{
215 *b += 1;
216 return a + *b;
217}
218
219u64 noinline bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d)
220{
221 return a + b + c + d;
222}
223
224int noinline bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b)
225{
226 return a + b;
227}
228
229struct sock * noinline bpf_kfunc_call_test3(struct sock *sk)
230{
231 return sk;
232}
233
234__diag_pop();
235
236ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO);
237
238BTF_SET_START(test_sk_kfunc_ids)
239BTF_ID(func, bpf_kfunc_call_test1)
240BTF_ID(func, bpf_kfunc_call_test2)
241BTF_ID(func, bpf_kfunc_call_test3)
242BTF_SET_END(test_sk_kfunc_ids)
243
244bool bpf_prog_test_check_kfunc_call(u32 kfunc_id)
245{
246 return btf_id_set_contains(&test_sk_kfunc_ids, kfunc_id);
247}
248
249static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
250 u32 headroom, u32 tailroom)
251{
252 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
253 u32 user_size = kattr->test.data_size_in;
254 void *data;
255
256 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
257 return ERR_PTR(-EINVAL);
258
259 if (user_size > size)
260 return ERR_PTR(-EMSGSIZE);
261
262 data = kzalloc(size + headroom + tailroom, GFP_USER);
263 if (!data)
264 return ERR_PTR(-ENOMEM);
265
266 if (copy_from_user(data + headroom, data_in, user_size)) {
267 kfree(data);
268 return ERR_PTR(-EFAULT);
269 }
270
271 return data;
272}
273
274int bpf_prog_test_run_tracing(struct bpf_prog *prog,
275 const union bpf_attr *kattr,
276 union bpf_attr __user *uattr)
277{
278 struct bpf_fentry_test_t arg = {};
279 u16 side_effect = 0, ret = 0;
280 int b = 2, err = -EFAULT;
281 u32 retval = 0;
282
283 if (kattr->test.flags || kattr->test.cpu)
284 return -EINVAL;
285
286 switch (prog->expected_attach_type) {
287 case BPF_TRACE_FENTRY:
288 case BPF_TRACE_FEXIT:
289 if (bpf_fentry_test1(1) != 2 ||
290 bpf_fentry_test2(2, 3) != 5 ||
291 bpf_fentry_test3(4, 5, 6) != 15 ||
292 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 ||
293 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 ||
294 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 ||
295 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 ||
296 bpf_fentry_test8(&arg) != 0)
297 goto out;
298 break;
299 case BPF_MODIFY_RETURN:
300 ret = bpf_modify_return_test(1, &b);
301 if (b != 2)
302 side_effect = 1;
303 break;
304 default:
305 goto out;
306 }
307
308 retval = ((u32)side_effect << 16) | ret;
309 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
310 goto out;
311
312 err = 0;
313out:
314 trace_bpf_test_finish(&err);
315 return err;
316}
317
318struct bpf_raw_tp_test_run_info {
319 struct bpf_prog *prog;
320 void *ctx;
321 u32 retval;
322};
323
324static void
325__bpf_prog_test_run_raw_tp(void *data)
326{
327 struct bpf_raw_tp_test_run_info *info = data;
328
329 rcu_read_lock();
330 info->retval = BPF_PROG_RUN(info->prog, info->ctx);
331 rcu_read_unlock();
332}
333
334int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
335 const union bpf_attr *kattr,
336 union bpf_attr __user *uattr)
337{
338 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
339 __u32 ctx_size_in = kattr->test.ctx_size_in;
340 struct bpf_raw_tp_test_run_info info;
341 int cpu = kattr->test.cpu, err = 0;
342 int current_cpu;
343
344
345 if (kattr->test.data_in || kattr->test.data_out ||
346 kattr->test.ctx_out || kattr->test.duration ||
347 kattr->test.repeat)
348 return -EINVAL;
349
350 if (ctx_size_in < prog->aux->max_ctx_offset ||
351 ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64))
352 return -EINVAL;
353
354 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
355 return -EINVAL;
356
357 if (ctx_size_in) {
358 info.ctx = kzalloc(ctx_size_in, GFP_USER);
359 if (!info.ctx)
360 return -ENOMEM;
361 if (copy_from_user(info.ctx, ctx_in, ctx_size_in)) {
362 err = -EFAULT;
363 goto out;
364 }
365 } else {
366 info.ctx = NULL;
367 }
368
369 info.prog = prog;
370
371 current_cpu = get_cpu();
372 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
373 cpu == current_cpu) {
374 __bpf_prog_test_run_raw_tp(&info);
375 } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
376
377
378
379
380
381 err = -ENXIO;
382 } else {
383 err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp,
384 &info, 1);
385 }
386 put_cpu();
387
388 if (!err &&
389 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32)))
390 err = -EFAULT;
391
392out:
393 kfree(info.ctx);
394 return err;
395}
396
397static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
398{
399 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
400 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
401 u32 size = kattr->test.ctx_size_in;
402 void *data;
403 int err;
404
405 if (!data_in && !data_out)
406 return NULL;
407
408 data = kzalloc(max_size, GFP_USER);
409 if (!data)
410 return ERR_PTR(-ENOMEM);
411
412 if (data_in) {
413 err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size);
414 if (err) {
415 kfree(data);
416 return ERR_PTR(err);
417 }
418
419 size = min_t(u32, max_size, size);
420 if (copy_from_user(data, data_in, size)) {
421 kfree(data);
422 return ERR_PTR(-EFAULT);
423 }
424 }
425 return data;
426}
427
428static int bpf_ctx_finish(const union bpf_attr *kattr,
429 union bpf_attr __user *uattr, const void *data,
430 u32 size)
431{
432 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
433 int err = -EFAULT;
434 u32 copy_size = size;
435
436 if (!data || !data_out)
437 return 0;
438
439 if (copy_size > kattr->test.ctx_size_out) {
440 copy_size = kattr->test.ctx_size_out;
441 err = -ENOSPC;
442 }
443
444 if (copy_to_user(data_out, data, copy_size))
445 goto out;
446 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
447 goto out;
448 if (err != -ENOSPC)
449 err = 0;
450out:
451 return err;
452}
453
454
455
456
457
458
459
460
461
462
463static inline bool range_is_zero(void *buf, size_t from, size_t to)
464{
465 return !memchr_inv((u8 *)buf + from, 0, to - from);
466}
467
468static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
469{
470 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
471
472 if (!__skb)
473 return 0;
474
475
476 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark)))
477 return -EINVAL;
478
479
480
481 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark),
482 offsetof(struct __sk_buff, priority)))
483 return -EINVAL;
484
485
486
487 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, priority),
488 offsetof(struct __sk_buff, ifindex)))
489 return -EINVAL;
490
491
492
493 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex),
494 offsetof(struct __sk_buff, cb)))
495 return -EINVAL;
496
497
498
499 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb),
500 offsetof(struct __sk_buff, tstamp)))
501 return -EINVAL;
502
503
504
505
506
507 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs),
508 offsetof(struct __sk_buff, gso_size)))
509 return -EINVAL;
510
511
512
513 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size),
514 sizeof(struct __sk_buff)))
515 return -EINVAL;
516
517 skb->mark = __skb->mark;
518 skb->priority = __skb->priority;
519 skb->tstamp = __skb->tstamp;
520 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
521
522 if (__skb->wire_len == 0) {
523 cb->pkt_len = skb->len;
524 } else {
525 if (__skb->wire_len < skb->len ||
526 __skb->wire_len > GSO_MAX_SIZE)
527 return -EINVAL;
528 cb->pkt_len = __skb->wire_len;
529 }
530
531 if (__skb->gso_segs > GSO_MAX_SEGS)
532 return -EINVAL;
533 skb_shinfo(skb)->gso_segs = __skb->gso_segs;
534 skb_shinfo(skb)->gso_size = __skb->gso_size;
535
536 return 0;
537}
538
539static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
540{
541 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
542
543 if (!__skb)
544 return;
545
546 __skb->mark = skb->mark;
547 __skb->priority = skb->priority;
548 __skb->ifindex = skb->dev->ifindex;
549 __skb->tstamp = skb->tstamp;
550 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
551 __skb->wire_len = cb->pkt_len;
552 __skb->gso_segs = skb_shinfo(skb)->gso_segs;
553}
554
555int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
556 union bpf_attr __user *uattr)
557{
558 bool is_l2 = false, is_direct_pkt_access = false;
559 struct net *net = current->nsproxy->net_ns;
560 struct net_device *dev = net->loopback_dev;
561 u32 size = kattr->test.data_size_in;
562 u32 repeat = kattr->test.repeat;
563 struct __sk_buff *ctx = NULL;
564 u32 retval, duration;
565 int hh_len = ETH_HLEN;
566 struct sk_buff *skb;
567 struct sock *sk;
568 void *data;
569 int ret;
570
571 if (kattr->test.flags || kattr->test.cpu)
572 return -EINVAL;
573
574 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
575 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
576 if (IS_ERR(data))
577 return PTR_ERR(data);
578
579 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
580 if (IS_ERR(ctx)) {
581 kfree(data);
582 return PTR_ERR(ctx);
583 }
584
585 switch (prog->type) {
586 case BPF_PROG_TYPE_SCHED_CLS:
587 case BPF_PROG_TYPE_SCHED_ACT:
588 is_l2 = true;
589 fallthrough;
590 case BPF_PROG_TYPE_LWT_IN:
591 case BPF_PROG_TYPE_LWT_OUT:
592 case BPF_PROG_TYPE_LWT_XMIT:
593 is_direct_pkt_access = true;
594 break;
595 default:
596 break;
597 }
598
599 sk = kzalloc(sizeof(struct sock), GFP_USER);
600 if (!sk) {
601 kfree(data);
602 kfree(ctx);
603 return -ENOMEM;
604 }
605 sock_net_set(sk, net);
606 sock_init_data(NULL, sk);
607
608 skb = build_skb(data, 0);
609 if (!skb) {
610 kfree(data);
611 kfree(ctx);
612 kfree(sk);
613 return -ENOMEM;
614 }
615 skb->sk = sk;
616
617 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
618 __skb_put(skb, size);
619 if (ctx && ctx->ifindex > 1) {
620 dev = dev_get_by_index(net, ctx->ifindex);
621 if (!dev) {
622 ret = -ENODEV;
623 goto out;
624 }
625 }
626 skb->protocol = eth_type_trans(skb, dev);
627 skb_reset_network_header(skb);
628
629 switch (skb->protocol) {
630 case htons(ETH_P_IP):
631 sk->sk_family = AF_INET;
632 if (sizeof(struct iphdr) <= skb_headlen(skb)) {
633 sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
634 sk->sk_daddr = ip_hdr(skb)->daddr;
635 }
636 break;
637#if IS_ENABLED(CONFIG_IPV6)
638 case htons(ETH_P_IPV6):
639 sk->sk_family = AF_INET6;
640 if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) {
641 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
642 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
643 }
644 break;
645#endif
646 default:
647 break;
648 }
649
650 if (is_l2)
651 __skb_push(skb, hh_len);
652 if (is_direct_pkt_access)
653 bpf_compute_data_pointers(skb);
654 ret = convert___skb_to_skb(skb, ctx);
655 if (ret)
656 goto out;
657 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false);
658 if (ret)
659 goto out;
660 if (!is_l2) {
661 if (skb_headroom(skb) < hh_len) {
662 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
663
664 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
665 ret = -ENOMEM;
666 goto out;
667 }
668 }
669 memset(__skb_push(skb, hh_len), 0, hh_len);
670 }
671 convert_skb_to___skb(skb, ctx);
672
673 size = skb->len;
674
675 if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
676 size = skb_headlen(skb);
677 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
678 if (!ret)
679 ret = bpf_ctx_finish(kattr, uattr, ctx,
680 sizeof(struct __sk_buff));
681out:
682 if (dev && dev != net->loopback_dev)
683 dev_put(dev);
684 kfree_skb(skb);
685 bpf_sk_storage_free(sk);
686 kfree(sk);
687 kfree(ctx);
688 return ret;
689}
690
691int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
692 union bpf_attr __user *uattr)
693{
694 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
695 u32 headroom = XDP_PACKET_HEADROOM;
696 u32 size = kattr->test.data_size_in;
697 u32 repeat = kattr->test.repeat;
698 struct netdev_rx_queue *rxqueue;
699 struct xdp_buff xdp = {};
700 u32 retval, duration;
701 u32 max_data_sz;
702 void *data;
703 int ret;
704
705 if (prog->expected_attach_type == BPF_XDP_DEVMAP ||
706 prog->expected_attach_type == BPF_XDP_CPUMAP)
707 return -EINVAL;
708 if (kattr->test.ctx_in || kattr->test.ctx_out)
709 return -EINVAL;
710
711
712 max_data_sz = 4096 - headroom - tailroom;
713
714 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom);
715 if (IS_ERR(data))
716 return PTR_ERR(data);
717
718 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
719 xdp_init_buff(&xdp, headroom + max_data_sz + tailroom,
720 &rxqueue->xdp_rxq);
721 xdp_prepare_buff(&xdp, data, headroom, size, true);
722
723 bpf_prog_change_xdp(NULL, prog);
724 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);
725 if (ret)
726 goto out;
727 if (xdp.data != data + headroom || xdp.data_end != xdp.data + size)
728 size = xdp.data_end - xdp.data;
729 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);
730out:
731 bpf_prog_change_xdp(prog, NULL);
732 kfree(data);
733 return ret;
734}
735
736static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
737{
738
739 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags)))
740 return -EINVAL;
741
742
743
744 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags),
745 sizeof(struct bpf_flow_keys)))
746 return -EINVAL;
747
748 return 0;
749}
750
751int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
752 const union bpf_attr *kattr,
753 union bpf_attr __user *uattr)
754{
755 struct bpf_test_timer t = { NO_PREEMPT };
756 u32 size = kattr->test.data_size_in;
757 struct bpf_flow_dissector ctx = {};
758 u32 repeat = kattr->test.repeat;
759 struct bpf_flow_keys *user_ctx;
760 struct bpf_flow_keys flow_keys;
761 const struct ethhdr *eth;
762 unsigned int flags = 0;
763 u32 retval, duration;
764 void *data;
765 int ret;
766
767 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
768 return -EINVAL;
769
770 if (kattr->test.flags || kattr->test.cpu)
771 return -EINVAL;
772
773 if (size < ETH_HLEN)
774 return -EINVAL;
775
776 data = bpf_test_init(kattr, size, 0, 0);
777 if (IS_ERR(data))
778 return PTR_ERR(data);
779
780 eth = (struct ethhdr *)data;
781
782 if (!repeat)
783 repeat = 1;
784
785 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));
786 if (IS_ERR(user_ctx)) {
787 kfree(data);
788 return PTR_ERR(user_ctx);
789 }
790 if (user_ctx) {
791 ret = verify_user_bpf_flow_keys(user_ctx);
792 if (ret)
793 goto out;
794 flags = user_ctx->flags;
795 }
796
797 ctx.flow_keys = &flow_keys;
798 ctx.data = data;
799 ctx.data_end = (__u8 *)data + size;
800
801 bpf_test_timer_enter(&t);
802 do {
803 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
804 size, flags);
805 } while (bpf_test_timer_continue(&t, repeat, &ret, &duration));
806 bpf_test_timer_leave(&t);
807
808 if (ret < 0)
809 goto out;
810
811 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
812 retval, duration);
813 if (!ret)
814 ret = bpf_ctx_finish(kattr, uattr, user_ctx,
815 sizeof(struct bpf_flow_keys));
816
817out:
818 kfree(user_ctx);
819 kfree(data);
820 return ret;
821}
822
823int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,
824 union bpf_attr __user *uattr)
825{
826 struct bpf_test_timer t = { NO_PREEMPT };
827 struct bpf_prog_array *progs = NULL;
828 struct bpf_sk_lookup_kern ctx = {};
829 u32 repeat = kattr->test.repeat;
830 struct bpf_sk_lookup *user_ctx;
831 u32 retval, duration;
832 int ret = -EINVAL;
833
834 if (prog->type != BPF_PROG_TYPE_SK_LOOKUP)
835 return -EINVAL;
836
837 if (kattr->test.flags || kattr->test.cpu)
838 return -EINVAL;
839
840 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||
841 kattr->test.data_size_out)
842 return -EINVAL;
843
844 if (!repeat)
845 repeat = 1;
846
847 user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx));
848 if (IS_ERR(user_ctx))
849 return PTR_ERR(user_ctx);
850
851 if (!user_ctx)
852 return -EINVAL;
853
854 if (user_ctx->sk)
855 goto out;
856
857 if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx)))
858 goto out;
859
860 if (user_ctx->local_port > U16_MAX || user_ctx->remote_port > U16_MAX) {
861 ret = -ERANGE;
862 goto out;
863 }
864
865 ctx.family = (u16)user_ctx->family;
866 ctx.protocol = (u16)user_ctx->protocol;
867 ctx.dport = (u16)user_ctx->local_port;
868 ctx.sport = (__force __be16)user_ctx->remote_port;
869
870 switch (ctx.family) {
871 case AF_INET:
872 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4;
873 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4;
874 break;
875
876#if IS_ENABLED(CONFIG_IPV6)
877 case AF_INET6:
878 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6;
879 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6;
880 break;
881#endif
882
883 default:
884 ret = -EAFNOSUPPORT;
885 goto out;
886 }
887
888 progs = bpf_prog_array_alloc(1, GFP_KERNEL);
889 if (!progs) {
890 ret = -ENOMEM;
891 goto out;
892 }
893
894 progs->items[0].prog = prog;
895
896 bpf_test_timer_enter(&t);
897 do {
898 ctx.selected_sk = NULL;
899 retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, BPF_PROG_RUN);
900 } while (bpf_test_timer_continue(&t, repeat, &ret, &duration));
901 bpf_test_timer_leave(&t);
902
903 if (ret < 0)
904 goto out;
905
906 user_ctx->cookie = 0;
907 if (ctx.selected_sk) {
908 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) {
909 ret = -EOPNOTSUPP;
910 goto out;
911 }
912
913 user_ctx->cookie = sock_gen_cookie(ctx.selected_sk);
914 }
915
916 ret = bpf_test_finish(kattr, uattr, NULL, 0, retval, duration);
917 if (!ret)
918 ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx));
919
920out:
921 bpf_prog_array_free(progs);
922 kfree(user_ctx);
923 return ret;
924}
925
926int bpf_prog_test_run_syscall(struct bpf_prog *prog,
927 const union bpf_attr *kattr,
928 union bpf_attr __user *uattr)
929{
930 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
931 __u32 ctx_size_in = kattr->test.ctx_size_in;
932 void *ctx = NULL;
933 u32 retval;
934 int err = 0;
935
936
937 if (kattr->test.data_in || kattr->test.data_out ||
938 kattr->test.ctx_out || kattr->test.duration ||
939 kattr->test.repeat || kattr->test.flags)
940 return -EINVAL;
941
942 if (ctx_size_in < prog->aux->max_ctx_offset ||
943 ctx_size_in > U16_MAX)
944 return -EINVAL;
945
946 if (ctx_size_in) {
947 ctx = kzalloc(ctx_size_in, GFP_USER);
948 if (!ctx)
949 return -ENOMEM;
950 if (copy_from_user(ctx, ctx_in, ctx_size_in)) {
951 err = -EFAULT;
952 goto out;
953 }
954 }
955
956 rcu_read_lock_trace();
957 retval = bpf_prog_run_pin_on_cpu(prog, ctx);
958 rcu_read_unlock_trace();
959
960 if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) {
961 err = -EFAULT;
962 goto out;
963 }
964 if (ctx_size_in)
965 if (copy_to_user(ctx_in, ctx, ctx_size_in))
966 err = -EFAULT;
967out:
968 kfree(ctx);
969 return err;
970}
971
