1
2
3
4#include <linux/bpf.h>
5#include <linux/btf.h>
6#include <linux/btf_ids.h>
7#include <linux/slab.h>
8#include <linux/vmalloc.h>
9#include <linux/etherdevice.h>
10#include <linux/filter.h>
11#include <linux/rcupdate_trace.h>
12#include <linux/sched/signal.h>
13#include <net/bpf_sk_storage.h>
14#include <net/sock.h>
15#include <net/tcp.h>
16#include <net/net_namespace.h>
17#include <linux/error-injection.h>
18#include <linux/smp.h>
19#include <linux/sock_diag.h>
20#include <net/xdp.h>
21
22#define CREATE_TRACE_POINTS
23#include <trace/events/bpf_test_run.h>
24
25struct bpf_test_timer {
26 enum { NO_PREEMPT, NO_MIGRATE } mode;
27 u32 i;
28 u64 time_start, time_spent;
29};
30
31static void bpf_test_timer_enter(struct bpf_test_timer *t)
32 __acquires(rcu)
33{
34 rcu_read_lock();
35 if (t->mode == NO_PREEMPT)
36 preempt_disable();
37 else
38 migrate_disable();
39
40 t->time_start = ktime_get_ns();
41}
42
43static void bpf_test_timer_leave(struct bpf_test_timer *t)
44 __releases(rcu)
45{
46 t->time_start = 0;
47
48 if (t->mode == NO_PREEMPT)
49 preempt_enable();
50 else
51 migrate_enable();
52 rcu_read_unlock();
53}
54
55static bool bpf_test_timer_continue(struct bpf_test_timer *t, u32 repeat, int *err, u32 *duration)
56 __must_hold(rcu)
57{
58 t->i++;
59 if (t->i >= repeat) {
60
61 t->time_spent += ktime_get_ns() - t->time_start;
62 do_div(t->time_spent, t->i);
63 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
64 *err = 0;
65 goto reset;
66 }
67
68 if (signal_pending(current)) {
69
70 *err = -EINTR;
71 goto reset;
72 }
73
74 if (need_resched()) {
75
76 t->time_spent += ktime_get_ns() - t->time_start;
77 bpf_test_timer_leave(t);
78 cond_resched();
79 bpf_test_timer_enter(t);
80 }
81
82
83 return true;
84
85reset:
86 t->i = 0;
87 return false;
88}
89
90static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
91 u32 *retval, u32 *time, bool xdp)
92{
93 struct bpf_prog_array_item item = {.prog = prog};
94 struct bpf_run_ctx *old_ctx;
95 struct bpf_cg_run_ctx run_ctx;
96 struct bpf_test_timer t = { NO_MIGRATE };
97 enum bpf_cgroup_storage_type stype;
98 int ret;
99
100 for_each_cgroup_storage_type(stype) {
101 item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
102 if (IS_ERR(item.cgroup_storage[stype])) {
103 item.cgroup_storage[stype] = NULL;
104 for_each_cgroup_storage_type(stype)
105 bpf_cgroup_storage_free(item.cgroup_storage[stype]);
106 return -ENOMEM;
107 }
108 }
109
110 if (!repeat)
111 repeat = 1;
112
113 bpf_test_timer_enter(&t);
114 old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
115 do {
116 run_ctx.prog_item = &item;
117 if (xdp)
118 *retval = bpf_prog_run_xdp(prog, ctx);
119 else
120 *retval = bpf_prog_run(prog, ctx);
121 } while (bpf_test_timer_continue(&t, repeat, &ret, time));
122 bpf_reset_run_ctx(old_ctx);
123 bpf_test_timer_leave(&t);
124
125 for_each_cgroup_storage_type(stype)
126 bpf_cgroup_storage_free(item.cgroup_storage[stype]);
127
128 return ret;
129}
130
131static int bpf_test_finish(const union bpf_attr *kattr,
132 union bpf_attr __user *uattr, const void *data,
133 u32 size, u32 retval, u32 duration)
134{
135 void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
136 int err = -EFAULT;
137 u32 copy_size = size;
138
139
140
141
142 if (kattr->test.data_size_out &&
143 copy_size > kattr->test.data_size_out) {
144 copy_size = kattr->test.data_size_out;
145 err = -ENOSPC;
146 }
147
148 if (data_out && copy_to_user(data_out, data, copy_size))
149 goto out;
150 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
151 goto out;
152 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
153 goto out;
154 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
155 goto out;
156 if (err != -ENOSPC)
157 err = 0;
158out:
159 trace_bpf_test_finish(&err);
160 return err;
161}
162
163
164
165
166
167__diag_push();
168__diag_ignore(GCC, 8, "-Wmissing-prototypes",
169 "Global functions as their definitions will be in vmlinux BTF");
170int noinline bpf_fentry_test1(int a)
171{
172 return a + 1;
173}
174
175int noinline bpf_fentry_test2(int a, u64 b)
176{
177 return a + b;
178}
179
180int noinline bpf_fentry_test3(char a, int b, u64 c)
181{
182 return a + b + c;
183}
184
185int noinline bpf_fentry_test4(void *a, char b, int c, u64 d)
186{
187 return (long)a + b + c + d;
188}
189
190int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e)
191{
192 return a + (long)b + c + d + e;
193}
194
195int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f)
196{
197 return a + (long)b + c + d + (long)e + f;
198}
199
200struct bpf_fentry_test_t {
201 struct bpf_fentry_test_t *a;
202};
203
204int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg)
205{
206 return (long)arg;
207}
208
209int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg)
210{
211 return (long)arg->a;
212}
213
214int noinline bpf_modify_return_test(int a, int *b)
215{
216 *b += 1;
217 return a + *b;
218}
219
220u64 noinline bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d)
221{
222 return a + b + c + d;
223}
224
225int noinline bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b)
226{
227 return a + b;
228}
229
230struct sock * noinline bpf_kfunc_call_test3(struct sock *sk)
231{
232 return sk;
233}
234
235__diag_pop();
236
237ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO);
238
239BTF_SET_START(test_sk_kfunc_ids)
240BTF_ID(func, bpf_kfunc_call_test1)
241BTF_ID(func, bpf_kfunc_call_test2)
242BTF_ID(func, bpf_kfunc_call_test3)
243BTF_SET_END(test_sk_kfunc_ids)
244
245bool bpf_prog_test_check_kfunc_call(u32 kfunc_id, struct module *owner)
246{
247 if (btf_id_set_contains(&test_sk_kfunc_ids, kfunc_id))
248 return true;
249 return bpf_check_mod_kfunc_call(&prog_test_kfunc_list, kfunc_id, owner);
250}
251
252static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
253 u32 headroom, u32 tailroom)
254{
255 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
256 u32 user_size = kattr->test.data_size_in;
257 void *data;
258
259 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
260 return ERR_PTR(-EINVAL);
261
262 if (user_size > size)
263 return ERR_PTR(-EMSGSIZE);
264
265 data = kzalloc(size + headroom + tailroom, GFP_USER);
266 if (!data)
267 return ERR_PTR(-ENOMEM);
268
269 if (copy_from_user(data + headroom, data_in, user_size)) {
270 kfree(data);
271 return ERR_PTR(-EFAULT);
272 }
273
274 return data;
275}
276
277int bpf_prog_test_run_tracing(struct bpf_prog *prog,
278 const union bpf_attr *kattr,
279 union bpf_attr __user *uattr)
280{
281 struct bpf_fentry_test_t arg = {};
282 u16 side_effect = 0, ret = 0;
283 int b = 2, err = -EFAULT;
284 u32 retval = 0;
285
286 if (kattr->test.flags || kattr->test.cpu)
287 return -EINVAL;
288
289 switch (prog->expected_attach_type) {
290 case BPF_TRACE_FENTRY:
291 case BPF_TRACE_FEXIT:
292 if (bpf_fentry_test1(1) != 2 ||
293 bpf_fentry_test2(2, 3) != 5 ||
294 bpf_fentry_test3(4, 5, 6) != 15 ||
295 bpf_fentry_test4((void *)7, 8, 9, 10) != 34 ||
296 bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 ||
297 bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 ||
298 bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 ||
299 bpf_fentry_test8(&arg) != 0)
300 goto out;
301 break;
302 case BPF_MODIFY_RETURN:
303 ret = bpf_modify_return_test(1, &b);
304 if (b != 2)
305 side_effect = 1;
306 break;
307 default:
308 goto out;
309 }
310
311 retval = ((u32)side_effect << 16) | ret;
312 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
313 goto out;
314
315 err = 0;
316out:
317 trace_bpf_test_finish(&err);
318 return err;
319}
320
321struct bpf_raw_tp_test_run_info {
322 struct bpf_prog *prog;
323 void *ctx;
324 u32 retval;
325};
326
327static void
328__bpf_prog_test_run_raw_tp(void *data)
329{
330 struct bpf_raw_tp_test_run_info *info = data;
331
332 rcu_read_lock();
333 info->retval = bpf_prog_run(info->prog, info->ctx);
334 rcu_read_unlock();
335}
336
337int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
338 const union bpf_attr *kattr,
339 union bpf_attr __user *uattr)
340{
341 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
342 __u32 ctx_size_in = kattr->test.ctx_size_in;
343 struct bpf_raw_tp_test_run_info info;
344 int cpu = kattr->test.cpu, err = 0;
345 int current_cpu;
346
347
348 if (kattr->test.data_in || kattr->test.data_out ||
349 kattr->test.ctx_out || kattr->test.duration ||
350 kattr->test.repeat)
351 return -EINVAL;
352
353 if (ctx_size_in < prog->aux->max_ctx_offset ||
354 ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64))
355 return -EINVAL;
356
357 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
358 return -EINVAL;
359
360 if (ctx_size_in) {
361 info.ctx = memdup_user(ctx_in, ctx_size_in);
362 if (IS_ERR(info.ctx))
363 return PTR_ERR(info.ctx);
364 } else {
365 info.ctx = NULL;
366 }
367
368 info.prog = prog;
369
370 current_cpu = get_cpu();
371 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
372 cpu == current_cpu) {
373 __bpf_prog_test_run_raw_tp(&info);
374 } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
375
376
377
378
379
380 err = -ENXIO;
381 } else {
382 err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp,
383 &info, 1);
384 }
385 put_cpu();
386
387 if (!err &&
388 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32)))
389 err = -EFAULT;
390
391 kfree(info.ctx);
392 return err;
393}
394
395static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
396{
397 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
398 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
399 u32 size = kattr->test.ctx_size_in;
400 void *data;
401 int err;
402
403 if (!data_in && !data_out)
404 return NULL;
405
406 data = kzalloc(max_size, GFP_USER);
407 if (!data)
408 return ERR_PTR(-ENOMEM);
409
410 if (data_in) {
411 err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size);
412 if (err) {
413 kfree(data);
414 return ERR_PTR(err);
415 }
416
417 size = min_t(u32, max_size, size);
418 if (copy_from_user(data, data_in, size)) {
419 kfree(data);
420 return ERR_PTR(-EFAULT);
421 }
422 }
423 return data;
424}
425
426static int bpf_ctx_finish(const union bpf_attr *kattr,
427 union bpf_attr __user *uattr, const void *data,
428 u32 size)
429{
430 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
431 int err = -EFAULT;
432 u32 copy_size = size;
433
434 if (!data || !data_out)
435 return 0;
436
437 if (copy_size > kattr->test.ctx_size_out) {
438 copy_size = kattr->test.ctx_size_out;
439 err = -ENOSPC;
440 }
441
442 if (copy_to_user(data_out, data, copy_size))
443 goto out;
444 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
445 goto out;
446 if (err != -ENOSPC)
447 err = 0;
448out:
449 return err;
450}
451
452
453
454
455
456
457
458
459
460
461static inline bool range_is_zero(void *buf, size_t from, size_t to)
462{
463 return !memchr_inv((u8 *)buf + from, 0, to - from);
464}
465
466static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
467{
468 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
469
470 if (!__skb)
471 return 0;
472
473
474 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark)))
475 return -EINVAL;
476
477
478
479 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark),
480 offsetof(struct __sk_buff, priority)))
481 return -EINVAL;
482
483
484
485
486
487 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex),
488 offsetof(struct __sk_buff, cb)))
489 return -EINVAL;
490
491
492
493 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb),
494 offsetof(struct __sk_buff, tstamp)))
495 return -EINVAL;
496
497
498
499
500
501 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs),
502 offsetof(struct __sk_buff, gso_size)))
503 return -EINVAL;
504
505
506
507 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size),
508 offsetof(struct __sk_buff, hwtstamp)))
509 return -EINVAL;
510
511
512
513 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp),
514 sizeof(struct __sk_buff)))
515 return -EINVAL;
516
517 skb->mark = __skb->mark;
518 skb->priority = __skb->priority;
519 skb->skb_iif = __skb->ingress_ifindex;
520 skb->tstamp = __skb->tstamp;
521 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
522
523 if (__skb->wire_len == 0) {
524 cb->pkt_len = skb->len;
525 } else {
526 if (__skb->wire_len < skb->len ||
527 __skb->wire_len > GSO_MAX_SIZE)
528 return -EINVAL;
529 cb->pkt_len = __skb->wire_len;
530 }
531
532 if (__skb->gso_segs > GSO_MAX_SEGS)
533 return -EINVAL;
534 skb_shinfo(skb)->gso_segs = __skb->gso_segs;
535 skb_shinfo(skb)->gso_size = __skb->gso_size;
536 skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp;
537
538 return 0;
539}
540
541static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
542{
543 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
544
545 if (!__skb)
546 return;
547
548 __skb->mark = skb->mark;
549 __skb->priority = skb->priority;
550 __skb->ingress_ifindex = skb->skb_iif;
551 __skb->ifindex = skb->dev->ifindex;
552 __skb->tstamp = skb->tstamp;
553 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
554 __skb->wire_len = cb->pkt_len;
555 __skb->gso_segs = skb_shinfo(skb)->gso_segs;
556 __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp;
557}
558
559static struct proto bpf_dummy_proto = {
560 .name = "bpf_dummy",
561 .owner = THIS_MODULE,
562 .obj_size = sizeof(struct sock),
563};
564
565int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
566 union bpf_attr __user *uattr)
567{
568 bool is_l2 = false, is_direct_pkt_access = false;
569 struct net *net = current->nsproxy->net_ns;
570 struct net_device *dev = net->loopback_dev;
571 u32 size = kattr->test.data_size_in;
572 u32 repeat = kattr->test.repeat;
573 struct __sk_buff *ctx = NULL;
574 u32 retval, duration;
575 int hh_len = ETH_HLEN;
576 struct sk_buff *skb;
577 struct sock *sk;
578 void *data;
579 int ret;
580
581 if (kattr->test.flags || kattr->test.cpu)
582 return -EINVAL;
583
584 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
585 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
586 if (IS_ERR(data))
587 return PTR_ERR(data);
588
589 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
590 if (IS_ERR(ctx)) {
591 kfree(data);
592 return PTR_ERR(ctx);
593 }
594
595 switch (prog->type) {
596 case BPF_PROG_TYPE_SCHED_CLS:
597 case BPF_PROG_TYPE_SCHED_ACT:
598 is_l2 = true;
599 fallthrough;
600 case BPF_PROG_TYPE_LWT_IN:
601 case BPF_PROG_TYPE_LWT_OUT:
602 case BPF_PROG_TYPE_LWT_XMIT:
603 is_direct_pkt_access = true;
604 break;
605 default:
606 break;
607 }
608
609 sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1);
610 if (!sk) {
611 kfree(data);
612 kfree(ctx);
613 return -ENOMEM;
614 }
615 sock_init_data(NULL, sk);
616
617 skb = build_skb(data, 0);
618 if (!skb) {
619 kfree(data);
620 kfree(ctx);
621 sk_free(sk);
622 return -ENOMEM;
623 }
624 skb->sk = sk;
625
626 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
627 __skb_put(skb, size);
628 if (ctx && ctx->ifindex > 1) {
629 dev = dev_get_by_index(net, ctx->ifindex);
630 if (!dev) {
631 ret = -ENODEV;
632 goto out;
633 }
634 }
635 skb->protocol = eth_type_trans(skb, dev);
636 skb_reset_network_header(skb);
637
638 switch (skb->protocol) {
639 case htons(ETH_P_IP):
640 sk->sk_family = AF_INET;
641 if (sizeof(struct iphdr) <= skb_headlen(skb)) {
642 sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
643 sk->sk_daddr = ip_hdr(skb)->daddr;
644 }
645 break;
646#if IS_ENABLED(CONFIG_IPV6)
647 case htons(ETH_P_IPV6):
648 sk->sk_family = AF_INET6;
649 if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) {
650 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
651 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
652 }
653 break;
654#endif
655 default:
656 break;
657 }
658
659 if (is_l2)
660 __skb_push(skb, hh_len);
661 if (is_direct_pkt_access)
662 bpf_compute_data_pointers(skb);
663 ret = convert___skb_to_skb(skb, ctx);
664 if (ret)
665 goto out;
666 ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false);
667 if (ret)
668 goto out;
669 if (!is_l2) {
670 if (skb_headroom(skb) < hh_len) {
671 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
672
673 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
674 ret = -ENOMEM;
675 goto out;
676 }
677 }
678 memset(__skb_push(skb, hh_len), 0, hh_len);
679 }
680 convert_skb_to___skb(skb, ctx);
681
682 size = skb->len;
683
684 if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
685 size = skb_headlen(skb);
686 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
687 if (!ret)
688 ret = bpf_ctx_finish(kattr, uattr, ctx,
689 sizeof(struct __sk_buff));
690out:
691 if (dev && dev != net->loopback_dev)
692 dev_put(dev);
693 kfree_skb(skb);
694 sk_free(sk);
695 kfree(ctx);
696 return ret;
697}
698
699static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)
700{
701 unsigned int ingress_ifindex, rx_queue_index;
702 struct netdev_rx_queue *rxqueue;
703 struct net_device *device;
704
705 if (!xdp_md)
706 return 0;
707
708 if (xdp_md->egress_ifindex != 0)
709 return -EINVAL;
710
711 ingress_ifindex = xdp_md->ingress_ifindex;
712 rx_queue_index = xdp_md->rx_queue_index;
713
714 if (!ingress_ifindex && rx_queue_index)
715 return -EINVAL;
716
717 if (ingress_ifindex) {
718 device = dev_get_by_index(current->nsproxy->net_ns,
719 ingress_ifindex);
720 if (!device)
721 return -ENODEV;
722
723 if (rx_queue_index >= device->real_num_rx_queues)
724 goto free_dev;
725
726 rxqueue = __netif_get_rx_queue(device, rx_queue_index);
727
728 if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq))
729 goto free_dev;
730
731 xdp->rxq = &rxqueue->xdp_rxq;
732
733
734
735 }
736
737 xdp->data = xdp->data_meta + xdp_md->data;
738 return 0;
739
740free_dev:
741 dev_put(device);
742 return -EINVAL;
743}
744
745static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)
746{
747 if (!xdp_md)
748 return;
749
750 xdp_md->data = xdp->data - xdp->data_meta;
751 xdp_md->data_end = xdp->data_end - xdp->data_meta;
752
753 if (xdp_md->ingress_ifindex)
754 dev_put(xdp->rxq->dev);
755}
756
757int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
758 union bpf_attr __user *uattr)
759{
760 u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
761 u32 headroom = XDP_PACKET_HEADROOM;
762 u32 size = kattr->test.data_size_in;
763 u32 repeat = kattr->test.repeat;
764 struct netdev_rx_queue *rxqueue;
765 struct xdp_buff xdp = {};
766 u32 retval, duration;
767 struct xdp_md *ctx;
768 u32 max_data_sz;
769 void *data;
770 int ret = -EINVAL;
771
772 if (prog->expected_attach_type == BPF_XDP_DEVMAP ||
773 prog->expected_attach_type == BPF_XDP_CPUMAP)
774 return -EINVAL;
775
776 ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md));
777 if (IS_ERR(ctx))
778 return PTR_ERR(ctx);
779
780 if (ctx) {
781
782 if (ctx->data_meta || ctx->data_end != size ||
783 ctx->data > ctx->data_end ||
784 unlikely(xdp_metalen_invalid(ctx->data)))
785 goto free_ctx;
786
787 headroom -= ctx->data;
788 }
789
790
791 max_data_sz = 4096 - headroom - tailroom;
792
793 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom);
794 if (IS_ERR(data)) {
795 ret = PTR_ERR(data);
796 goto free_ctx;
797 }
798
799 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
800 xdp_init_buff(&xdp, headroom + max_data_sz + tailroom,
801 &rxqueue->xdp_rxq);
802 xdp_prepare_buff(&xdp, data, headroom, size, true);
803
804 ret = xdp_convert_md_to_buff(ctx, &xdp);
805 if (ret)
806 goto free_data;
807
808 if (repeat > 1)
809 bpf_prog_change_xdp(NULL, prog);
810 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);
811
812
813
814
815 xdp_convert_buff_to_md(&xdp, ctx);
816 if (ret)
817 goto out;
818
819 if (xdp.data_meta != data + headroom ||
820 xdp.data_end != xdp.data_meta + size)
821 size = xdp.data_end - xdp.data_meta;
822
823 ret = bpf_test_finish(kattr, uattr, xdp.data_meta, size, retval,
824 duration);
825 if (!ret)
826 ret = bpf_ctx_finish(kattr, uattr, ctx,
827 sizeof(struct xdp_md));
828
829out:
830 if (repeat > 1)
831 bpf_prog_change_xdp(prog, NULL);
832free_data:
833 kfree(data);
834free_ctx:
835 kfree(ctx);
836 return ret;
837}
838
839static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
840{
841
842 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags)))
843 return -EINVAL;
844
845
846
847 if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags),
848 sizeof(struct bpf_flow_keys)))
849 return -EINVAL;
850
851 return 0;
852}
853
854int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
855 const union bpf_attr *kattr,
856 union bpf_attr __user *uattr)
857{
858 struct bpf_test_timer t = { NO_PREEMPT };
859 u32 size = kattr->test.data_size_in;
860 struct bpf_flow_dissector ctx = {};
861 u32 repeat = kattr->test.repeat;
862 struct bpf_flow_keys *user_ctx;
863 struct bpf_flow_keys flow_keys;
864 const struct ethhdr *eth;
865 unsigned int flags = 0;
866 u32 retval, duration;
867 void *data;
868 int ret;
869
870 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
871 return -EINVAL;
872
873 if (kattr->test.flags || kattr->test.cpu)
874 return -EINVAL;
875
876 if (size < ETH_HLEN)
877 return -EINVAL;
878
879 data = bpf_test_init(kattr, size, 0, 0);
880 if (IS_ERR(data))
881 return PTR_ERR(data);
882
883 eth = (struct ethhdr *)data;
884
885 if (!repeat)
886 repeat = 1;
887
888 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));
889 if (IS_ERR(user_ctx)) {
890 kfree(data);
891 return PTR_ERR(user_ctx);
892 }
893 if (user_ctx) {
894 ret = verify_user_bpf_flow_keys(user_ctx);
895 if (ret)
896 goto out;
897 flags = user_ctx->flags;
898 }
899
900 ctx.flow_keys = &flow_keys;
901 ctx.data = data;
902 ctx.data_end = (__u8 *)data + size;
903
904 bpf_test_timer_enter(&t);
905 do {
906 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
907 size, flags);
908 } while (bpf_test_timer_continue(&t, repeat, &ret, &duration));
909 bpf_test_timer_leave(&t);
910
911 if (ret < 0)
912 goto out;
913
914 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
915 retval, duration);
916 if (!ret)
917 ret = bpf_ctx_finish(kattr, uattr, user_ctx,
918 sizeof(struct bpf_flow_keys));
919
920out:
921 kfree(user_ctx);
922 kfree(data);
923 return ret;
924}
925
926int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,
927 union bpf_attr __user *uattr)
928{
929 struct bpf_test_timer t = { NO_PREEMPT };
930 struct bpf_prog_array *progs = NULL;
931 struct bpf_sk_lookup_kern ctx = {};
932 u32 repeat = kattr->test.repeat;
933 struct bpf_sk_lookup *user_ctx;
934 u32 retval, duration;
935 int ret = -EINVAL;
936
937 if (prog->type != BPF_PROG_TYPE_SK_LOOKUP)
938 return -EINVAL;
939
940 if (kattr->test.flags || kattr->test.cpu)
941 return -EINVAL;
942
943 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||
944 kattr->test.data_size_out)
945 return -EINVAL;
946
947 if (!repeat)
948 repeat = 1;
949
950 user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx));
951 if (IS_ERR(user_ctx))
952 return PTR_ERR(user_ctx);
953
954 if (!user_ctx)
955 return -EINVAL;
956
957 if (user_ctx->sk)
958 goto out;
959
960 if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx)))
961 goto out;
962
963 if (user_ctx->local_port > U16_MAX || user_ctx->remote_port > U16_MAX) {
964 ret = -ERANGE;
965 goto out;
966 }
967
968 ctx.family = (u16)user_ctx->family;
969 ctx.protocol = (u16)user_ctx->protocol;
970 ctx.dport = (u16)user_ctx->local_port;
971 ctx.sport = (__force __be16)user_ctx->remote_port;
972
973 switch (ctx.family) {
974 case AF_INET:
975 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4;
976 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4;
977 break;
978
979#if IS_ENABLED(CONFIG_IPV6)
980 case AF_INET6:
981 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6;
982 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6;
983 break;
984#endif
985
986 default:
987 ret = -EAFNOSUPPORT;
988 goto out;
989 }
990
991 progs = bpf_prog_array_alloc(1, GFP_KERNEL);
992 if (!progs) {
993 ret = -ENOMEM;
994 goto out;
995 }
996
997 progs->items[0].prog = prog;
998
999 bpf_test_timer_enter(&t);
1000 do {
1001 ctx.selected_sk = NULL;
1002 retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run);
1003 } while (bpf_test_timer_continue(&t, repeat, &ret, &duration));
1004 bpf_test_timer_leave(&t);
1005
1006 if (ret < 0)
1007 goto out;
1008
1009 user_ctx->cookie = 0;
1010 if (ctx.selected_sk) {
1011 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) {
1012 ret = -EOPNOTSUPP;
1013 goto out;
1014 }
1015
1016 user_ctx->cookie = sock_gen_cookie(ctx.selected_sk);
1017 }
1018
1019 ret = bpf_test_finish(kattr, uattr, NULL, 0, retval, duration);
1020 if (!ret)
1021 ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx));
1022
1023out:
1024 bpf_prog_array_free(progs);
1025 kfree(user_ctx);
1026 return ret;
1027}
1028
1029int bpf_prog_test_run_syscall(struct bpf_prog *prog,
1030 const union bpf_attr *kattr,
1031 union bpf_attr __user *uattr)
1032{
1033 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
1034 __u32 ctx_size_in = kattr->test.ctx_size_in;
1035 void *ctx = NULL;
1036 u32 retval;
1037 int err = 0;
1038
1039
1040 if (kattr->test.data_in || kattr->test.data_out ||
1041 kattr->test.ctx_out || kattr->test.duration ||
1042 kattr->test.repeat || kattr->test.flags)
1043 return -EINVAL;
1044
1045 if (ctx_size_in < prog->aux->max_ctx_offset ||
1046 ctx_size_in > U16_MAX)
1047 return -EINVAL;
1048
1049 if (ctx_size_in) {
1050 ctx = memdup_user(ctx_in, ctx_size_in);
1051 if (IS_ERR(ctx))
1052 return PTR_ERR(ctx);
1053 }
1054
1055 rcu_read_lock_trace();
1056 retval = bpf_prog_run_pin_on_cpu(prog, ctx);
1057 rcu_read_unlock_trace();
1058
1059 if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) {
1060 err = -EFAULT;
1061 goto out;
1062 }
1063 if (ctx_size_in)
1064 if (copy_to_user(ctx_in, ctx, ctx_size_in))
1065 err = -EFAULT;
1066out:
1067 kfree(ctx);
1068 return err;
1069}
1070
