1
2
3
4
5
6
7#include <linux/bpf.h>
8#include <linux/slab.h>
9#include <linux/vmalloc.h>
10#include <linux/etherdevice.h>
11#include <linux/filter.h>
12#include <linux/sched/signal.h>
13#include <net/sock.h>
14#include <net/tcp.h>
15
16static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
17 u32 *retval, u32 *time)
18{
19 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL };
20 enum bpf_cgroup_storage_type stype;
21 u64 time_start, time_spent = 0;
22 int ret = 0;
23 u32 i;
24
25 for_each_cgroup_storage_type(stype) {
26 storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
27 if (IS_ERR(storage[stype])) {
28 storage[stype] = NULL;
29 for_each_cgroup_storage_type(stype)
30 bpf_cgroup_storage_free(storage[stype]);
31 return -ENOMEM;
32 }
33 }
34
35 if (!repeat)
36 repeat = 1;
37
38 rcu_read_lock();
39 preempt_disable();
40 time_start = ktime_get_ns();
41 for (i = 0; i < repeat; i++) {
42 bpf_cgroup_storage_set(storage);
43 *retval = BPF_PROG_RUN(prog, ctx);
44
45 if (signal_pending(current)) {
46 ret = -EINTR;
47 break;
48 }
49
50 if (need_resched()) {
51 time_spent += ktime_get_ns() - time_start;
52 preempt_enable();
53 rcu_read_unlock();
54
55 cond_resched();
56
57 rcu_read_lock();
58 preempt_disable();
59 time_start = ktime_get_ns();
60 }
61 }
62 time_spent += ktime_get_ns() - time_start;
63 preempt_enable();
64 rcu_read_unlock();
65
66 do_div(time_spent, repeat);
67 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
68
69 for_each_cgroup_storage_type(stype)
70 bpf_cgroup_storage_free(storage[stype]);
71
72 return ret;
73}
74
75static int bpf_test_finish(const union bpf_attr *kattr,
76 union bpf_attr __user *uattr, const void *data,
77 u32 size, u32 retval, u32 duration)
78{
79 void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
80 int err = -EFAULT;
81 u32 copy_size = size;
82
83
84
85
86 if (kattr->test.data_size_out &&
87 copy_size > kattr->test.data_size_out) {
88 copy_size = kattr->test.data_size_out;
89 err = -ENOSPC;
90 }
91
92 if (data_out && copy_to_user(data_out, data, copy_size))
93 goto out;
94 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
95 goto out;
96 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
97 goto out;
98 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
99 goto out;
100 if (err != -ENOSPC)
101 err = 0;
102out:
103 return err;
104}
105
106static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
107 u32 headroom, u32 tailroom)
108{
109 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
110 void *data;
111
112 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
113 return ERR_PTR(-EINVAL);
114
115 data = kzalloc(size + headroom + tailroom, GFP_USER);
116 if (!data)
117 return ERR_PTR(-ENOMEM);
118
119 if (copy_from_user(data + headroom, data_in, size)) {
120 kfree(data);
121 return ERR_PTR(-EFAULT);
122 }
123 return data;
124}
125
126int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
127 union bpf_attr __user *uattr)
128{
129 bool is_l2 = false, is_direct_pkt_access = false;
130 u32 size = kattr->test.data_size_in;
131 u32 repeat = kattr->test.repeat;
132 u32 retval, duration;
133 int hh_len = ETH_HLEN;
134 struct sk_buff *skb;
135 struct sock *sk;
136 void *data;
137 int ret;
138
139 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
140 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
141 if (IS_ERR(data))
142 return PTR_ERR(data);
143
144 switch (prog->type) {
145 case BPF_PROG_TYPE_SCHED_CLS:
146 case BPF_PROG_TYPE_SCHED_ACT:
147 is_l2 = true;
148
149 case BPF_PROG_TYPE_LWT_IN:
150 case BPF_PROG_TYPE_LWT_OUT:
151 case BPF_PROG_TYPE_LWT_XMIT:
152 is_direct_pkt_access = true;
153 break;
154 default:
155 break;
156 }
157
158 sk = kzalloc(sizeof(struct sock), GFP_USER);
159 if (!sk) {
160 kfree(data);
161 return -ENOMEM;
162 }
163 sock_net_set(sk, current->nsproxy->net_ns);
164 sock_init_data(NULL, sk);
165
166 skb = build_skb(data, 0);
167 if (!skb) {
168 kfree(data);
169 kfree(sk);
170 return -ENOMEM;
171 }
172 skb->sk = sk;
173
174 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
175 __skb_put(skb, size);
176 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev);
177 skb_reset_network_header(skb);
178
179 if (is_l2)
180 __skb_push(skb, hh_len);
181 if (is_direct_pkt_access)
182 bpf_compute_data_pointers(skb);
183 ret = bpf_test_run(prog, skb, repeat, &retval, &duration);
184 if (ret) {
185 kfree_skb(skb);
186 kfree(sk);
187 return ret;
188 }
189 if (!is_l2) {
190 if (skb_headroom(skb) < hh_len) {
191 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
192
193 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
194 kfree_skb(skb);
195 kfree(sk);
196 return -ENOMEM;
197 }
198 }
199 memset(__skb_push(skb, hh_len), 0, hh_len);
200 }
201
202 size = skb->len;
203
204 if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
205 size = skb_headlen(skb);
206 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
207 kfree_skb(skb);
208 kfree(sk);
209 return ret;
210}
211
212int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
213 union bpf_attr __user *uattr)
214{
215 u32 size = kattr->test.data_size_in;
216 u32 repeat = kattr->test.repeat;
217 struct netdev_rx_queue *rxqueue;
218 struct xdp_buff xdp = {};
219 u32 retval, duration;
220 void *data;
221 int ret;
222
223 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0);
224 if (IS_ERR(data))
225 return PTR_ERR(data);
226
227 xdp.data_hard_start = data;
228 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN;
229 xdp.data_meta = xdp.data;
230 xdp.data_end = xdp.data + size;
231
232 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
233 xdp.rxq = &rxqueue->xdp_rxq;
234
235 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration);
236 if (ret)
237 goto out;
238 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN ||
239 xdp.data_end != xdp.data + size)
240 size = xdp.data_end - xdp.data;
241 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);
242out:
243 kfree(data);
244 return ret;
245}
246
247int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
248 const union bpf_attr *kattr,
249 union bpf_attr __user *uattr)
250{
251 u32 size = kattr->test.data_size_in;
252 u32 repeat = kattr->test.repeat;
253 struct bpf_flow_keys flow_keys;
254 u64 time_start, time_spent = 0;
255 struct bpf_skb_data_end *cb;
256 u32 retval, duration;
257 struct sk_buff *skb;
258 struct sock *sk;
259 void *data;
260 int ret;
261 u32 i;
262
263 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
264 return -EINVAL;
265
266 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
267 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
268 if (IS_ERR(data))
269 return PTR_ERR(data);
270
271 sk = kzalloc(sizeof(*sk), GFP_USER);
272 if (!sk) {
273 kfree(data);
274 return -ENOMEM;
275 }
276 sock_net_set(sk, current->nsproxy->net_ns);
277 sock_init_data(NULL, sk);
278
279 skb = build_skb(data, 0);
280 if (!skb) {
281 kfree(data);
282 kfree(sk);
283 return -ENOMEM;
284 }
285 skb->sk = sk;
286
287 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
288 __skb_put(skb, size);
289 skb->protocol = eth_type_trans(skb,
290 current->nsproxy->net_ns->loopback_dev);
291 skb_reset_network_header(skb);
292
293 cb = (struct bpf_skb_data_end *)skb->cb;
294 cb->qdisc_cb.flow_keys = &flow_keys;
295
296 if (!repeat)
297 repeat = 1;
298
299 rcu_read_lock();
300 preempt_disable();
301 time_start = ktime_get_ns();
302 for (i = 0; i < repeat; i++) {
303 retval = __skb_flow_bpf_dissect(prog, skb,
304 &flow_keys_dissector,
305 &flow_keys);
306
307 if (signal_pending(current)) {
308 preempt_enable();
309 rcu_read_unlock();
310
311 ret = -EINTR;
312 goto out;
313 }
314
315 if (need_resched()) {
316 time_spent += ktime_get_ns() - time_start;
317 preempt_enable();
318 rcu_read_unlock();
319
320 cond_resched();
321
322 rcu_read_lock();
323 preempt_disable();
324 time_start = ktime_get_ns();
325 }
326 }
327 time_spent += ktime_get_ns() - time_start;
328 preempt_enable();
329 rcu_read_unlock();
330
331 do_div(time_spent, repeat);
332 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
333
334 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
335 retval, duration);
336
337out:
338 kfree_skb(skb);
339 kfree(sk);
340 return ret;
341}
342
