1
2
3
4#include <linux/bpf.h>
5#include <linux/slab.h>
6#include <linux/vmalloc.h>
7#include <linux/etherdevice.h>
8#include <linux/filter.h>
9#include <linux/sched/signal.h>
10#include <net/bpf_sk_storage.h>
11#include <net/sock.h>
12#include <net/tcp.h>
13
14#define CREATE_TRACE_POINTS
15#include <trace/events/bpf_test_run.h>
16
17static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
18 u32 *retval, u32 *time)
19{
20 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL };
21 enum bpf_cgroup_storage_type stype;
22 u64 time_start, time_spent = 0;
23 int ret = 0;
24 u32 i;
25
26 for_each_cgroup_storage_type(stype) {
27 storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
28 if (IS_ERR(storage[stype])) {
29 storage[stype] = NULL;
30 for_each_cgroup_storage_type(stype)
31 bpf_cgroup_storage_free(storage[stype]);
32 return -ENOMEM;
33 }
34 }
35
36 if (!repeat)
37 repeat = 1;
38
39 rcu_read_lock();
40 preempt_disable();
41 time_start = ktime_get_ns();
42 for (i = 0; i < repeat; i++) {
43 bpf_cgroup_storage_set(storage);
44 *retval = BPF_PROG_RUN(prog, ctx);
45
46 if (signal_pending(current)) {
47 ret = -EINTR;
48 break;
49 }
50
51 if (need_resched()) {
52 time_spent += ktime_get_ns() - time_start;
53 preempt_enable();
54 rcu_read_unlock();
55
56 cond_resched();
57
58 rcu_read_lock();
59 preempt_disable();
60 time_start = ktime_get_ns();
61 }
62 }
63 time_spent += ktime_get_ns() - time_start;
64 preempt_enable();
65 rcu_read_unlock();
66
67 do_div(time_spent, repeat);
68 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
69
70 for_each_cgroup_storage_type(stype)
71 bpf_cgroup_storage_free(storage[stype]);
72
73 return ret;
74}
75
76static int bpf_test_finish(const union bpf_attr *kattr,
77 union bpf_attr __user *uattr, const void *data,
78 u32 size, u32 retval, u32 duration)
79{
80 void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
81 int err = -EFAULT;
82 u32 copy_size = size;
83
84
85
86
87 if (kattr->test.data_size_out &&
88 copy_size > kattr->test.data_size_out) {
89 copy_size = kattr->test.data_size_out;
90 err = -ENOSPC;
91 }
92
93 if (data_out && copy_to_user(data_out, data, copy_size))
94 goto out;
95 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
96 goto out;
97 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
98 goto out;
99 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
100 goto out;
101 if (err != -ENOSPC)
102 err = 0;
103out:
104 trace_bpf_test_finish(&err);
105 return err;
106}
107
108static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
109 u32 headroom, u32 tailroom)
110{
111 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
112 void *data;
113
114 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
115 return ERR_PTR(-EINVAL);
116
117 data = kzalloc(size + headroom + tailroom, GFP_USER);
118 if (!data)
119 return ERR_PTR(-ENOMEM);
120
121 if (copy_from_user(data + headroom, data_in, size)) {
122 kfree(data);
123 return ERR_PTR(-EFAULT);
124 }
125 return data;
126}
127
128static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
129{
130 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
131 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
132 u32 size = kattr->test.ctx_size_in;
133 void *data;
134 int err;
135
136 if (!data_in && !data_out)
137 return NULL;
138
139 data = kzalloc(max_size, GFP_USER);
140 if (!data)
141 return ERR_PTR(-ENOMEM);
142
143 if (data_in) {
144 err = bpf_check_uarg_tail_zero(data_in, max_size, size);
145 if (err) {
146 kfree(data);
147 return ERR_PTR(err);
148 }
149
150 size = min_t(u32, max_size, size);
151 if (copy_from_user(data, data_in, size)) {
152 kfree(data);
153 return ERR_PTR(-EFAULT);
154 }
155 }
156 return data;
157}
158
159static int bpf_ctx_finish(const union bpf_attr *kattr,
160 union bpf_attr __user *uattr, const void *data,
161 u32 size)
162{
163 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
164 int err = -EFAULT;
165 u32 copy_size = size;
166
167 if (!data || !data_out)
168 return 0;
169
170 if (copy_size > kattr->test.ctx_size_out) {
171 copy_size = kattr->test.ctx_size_out;
172 err = -ENOSPC;
173 }
174
175 if (copy_to_user(data_out, data, copy_size))
176 goto out;
177 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
178 goto out;
179 if (err != -ENOSPC)
180 err = 0;
181out:
182 return err;
183}
184
185
186
187
188
189
190
191
192
193
194static inline bool range_is_zero(void *buf, size_t from, size_t to)
195{
196 return !memchr_inv((u8 *)buf + from, 0, to - from);
197}
198
199static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
200{
201 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
202
203 if (!__skb)
204 return 0;
205
206
207 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, priority)))
208 return -EINVAL;
209
210
211
212 if (!range_is_zero(__skb, offsetof(struct __sk_buff, priority) +
213 FIELD_SIZEOF(struct __sk_buff, priority),
214 offsetof(struct __sk_buff, cb)))
215 return -EINVAL;
216
217
218
219 if (!range_is_zero(__skb, offsetof(struct __sk_buff, cb) +
220 FIELD_SIZEOF(struct __sk_buff, cb),
221 sizeof(struct __sk_buff)))
222 return -EINVAL;
223
224 skb->priority = __skb->priority;
225 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
226
227 return 0;
228}
229
230static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
231{
232 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
233
234 if (!__skb)
235 return;
236
237 __skb->priority = skb->priority;
238 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
239}
240
241int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
242 union bpf_attr __user *uattr)
243{
244 bool is_l2 = false, is_direct_pkt_access = false;
245 u32 size = kattr->test.data_size_in;
246 u32 repeat = kattr->test.repeat;
247 struct __sk_buff *ctx = NULL;
248 u32 retval, duration;
249 int hh_len = ETH_HLEN;
250 struct sk_buff *skb;
251 struct sock *sk;
252 void *data;
253 int ret;
254
255 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
256 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
257 if (IS_ERR(data))
258 return PTR_ERR(data);
259
260 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
261 if (IS_ERR(ctx)) {
262 kfree(data);
263 return PTR_ERR(ctx);
264 }
265
266 switch (prog->type) {
267 case BPF_PROG_TYPE_SCHED_CLS:
268 case BPF_PROG_TYPE_SCHED_ACT:
269 is_l2 = true;
270
271 case BPF_PROG_TYPE_LWT_IN:
272 case BPF_PROG_TYPE_LWT_OUT:
273 case BPF_PROG_TYPE_LWT_XMIT:
274 is_direct_pkt_access = true;
275 break;
276 default:
277 break;
278 }
279
280 sk = kzalloc(sizeof(struct sock), GFP_USER);
281 if (!sk) {
282 kfree(data);
283 kfree(ctx);
284 return -ENOMEM;
285 }
286 sock_net_set(sk, current->nsproxy->net_ns);
287 sock_init_data(NULL, sk);
288
289 skb = build_skb(data, 0);
290 if (!skb) {
291 kfree(data);
292 kfree(ctx);
293 kfree(sk);
294 return -ENOMEM;
295 }
296 skb->sk = sk;
297
298 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
299 __skb_put(skb, size);
300 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev);
301 skb_reset_network_header(skb);
302
303 if (is_l2)
304 __skb_push(skb, hh_len);
305 if (is_direct_pkt_access)
306 bpf_compute_data_pointers(skb);
307 ret = convert___skb_to_skb(skb, ctx);
308 if (ret)
309 goto out;
310 ret = bpf_test_run(prog, skb, repeat, &retval, &duration);
311 if (ret)
312 goto out;
313 if (!is_l2) {
314 if (skb_headroom(skb) < hh_len) {
315 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
316
317 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
318 ret = -ENOMEM;
319 goto out;
320 }
321 }
322 memset(__skb_push(skb, hh_len), 0, hh_len);
323 }
324 convert_skb_to___skb(skb, ctx);
325
326 size = skb->len;
327
328 if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
329 size = skb_headlen(skb);
330 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
331 if (!ret)
332 ret = bpf_ctx_finish(kattr, uattr, ctx,
333 sizeof(struct __sk_buff));
334out:
335 kfree_skb(skb);
336 bpf_sk_storage_free(sk);
337 kfree(sk);
338 kfree(ctx);
339 return ret;
340}
341
342int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
343 union bpf_attr __user *uattr)
344{
345 u32 size = kattr->test.data_size_in;
346 u32 repeat = kattr->test.repeat;
347 struct netdev_rx_queue *rxqueue;
348 struct xdp_buff xdp = {};
349 u32 retval, duration;
350 void *data;
351 int ret;
352
353 if (kattr->test.ctx_in || kattr->test.ctx_out)
354 return -EINVAL;
355
356 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0);
357 if (IS_ERR(data))
358 return PTR_ERR(data);
359
360 xdp.data_hard_start = data;
361 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN;
362 xdp.data_meta = xdp.data;
363 xdp.data_end = xdp.data + size;
364
365 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
366 xdp.rxq = &rxqueue->xdp_rxq;
367
368 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration);
369 if (ret)
370 goto out;
371 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN ||
372 xdp.data_end != xdp.data + size)
373 size = xdp.data_end - xdp.data;
374 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);
375out:
376 kfree(data);
377 return ret;
378}
379
380int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
381 const union bpf_attr *kattr,
382 union bpf_attr __user *uattr)
383{
384 u32 size = kattr->test.data_size_in;
385 struct bpf_flow_dissector ctx = {};
386 u32 repeat = kattr->test.repeat;
387 struct bpf_flow_keys flow_keys;
388 u64 time_start, time_spent = 0;
389 const struct ethhdr *eth;
390 u32 retval, duration;
391 void *data;
392 int ret;
393 u32 i;
394
395 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
396 return -EINVAL;
397
398 if (kattr->test.ctx_in || kattr->test.ctx_out)
399 return -EINVAL;
400
401 if (size < ETH_HLEN)
402 return -EINVAL;
403
404 data = bpf_test_init(kattr, size, 0, 0);
405 if (IS_ERR(data))
406 return PTR_ERR(data);
407
408 eth = (struct ethhdr *)data;
409
410 if (!repeat)
411 repeat = 1;
412
413 ctx.flow_keys = &flow_keys;
414 ctx.data = data;
415 ctx.data_end = (__u8 *)data + size;
416
417 rcu_read_lock();
418 preempt_disable();
419 time_start = ktime_get_ns();
420 for (i = 0; i < repeat; i++) {
421 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
422 size);
423
424 if (signal_pending(current)) {
425 preempt_enable();
426 rcu_read_unlock();
427
428 ret = -EINTR;
429 goto out;
430 }
431
432 if (need_resched()) {
433 time_spent += ktime_get_ns() - time_start;
434 preempt_enable();
435 rcu_read_unlock();
436
437 cond_resched();
438
439 rcu_read_lock();
440 preempt_disable();
441 time_start = ktime_get_ns();
442 }
443 }
444 time_spent += ktime_get_ns() - time_start;
445 preempt_enable();
446 rcu_read_unlock();
447
448 do_div(time_spent, repeat);
449 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
450
451 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
452 retval, duration);
453
454out:
455 kfree(data);
456 return ret;
457}
458
