1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/module.h>
20#include <linux/types.h>
21#include <linux/mm.h>
22#include <linux/fcntl.h>
23#include <linux/socket.h>
24#include <linux/in.h>
25#include <linux/inet.h>
26#include <linux/netdevice.h>
27#include <linux/if_packet.h>
28#include <linux/gfp.h>
29#include <net/ip.h>
30#include <net/protocol.h>
31#include <net/netlink.h>
32#include <linux/skbuff.h>
33#include <net/sock.h>
34#include <linux/errno.h>
35#include <linux/timer.h>
36#include <asm/uaccess.h>
37#include <asm/unaligned.h>
38#include <linux/filter.h>
39#include <linux/reciprocal_div.h>
40#include <linux/ratelimit.h>
41
42
43
44
45
46void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size)
47{
48 u8 *ptr = NULL;
49
50 if (k >= SKF_NET_OFF)
51 ptr = skb_network_header(skb) + k - SKF_NET_OFF;
52 else if (k >= SKF_LL_OFF)
53 ptr = skb_mac_header(skb) + k - SKF_LL_OFF;
54
55 if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb))
56 return ptr;
57 return NULL;
58}
59
60static inline void *load_pointer(const struct sk_buff *skb, int k,
61 unsigned int size, void *buffer)
62{
63 if (k >= 0)
64 return skb_header_pointer(skb, k, size, buffer);
65 return bpf_internal_load_pointer_neg_helper(skb, k, size);
66}
67
68
69
70
71
72
73
74
75
76
77
78
79
80int sk_filter(struct sock *sk, struct sk_buff *skb)
81{
82 int err;
83 struct sk_filter *filter;
84
85 err = security_sock_rcv_skb(sk, skb);
86 if (err)
87 return err;
88
89 rcu_read_lock();
90 filter = rcu_dereference(sk->sk_filter);
91 if (filter) {
92 unsigned int pkt_len = SK_RUN_FILTER(filter, skb);
93
94 err = pkt_len ? pskb_trim(skb, pkt_len) : -EPERM;
95 }
96 rcu_read_unlock();
97
98 return err;
99}
100EXPORT_SYMBOL(sk_filter);
101
102
103
104
105
106
107
108
109
110
111
112
113
114unsigned int sk_run_filter(const struct sk_buff *skb,
115 const struct sock_filter *fentry)
116{
117 void *ptr;
118 u32 A = 0;
119 u32 X = 0;
120 u32 mem[BPF_MEMWORDS];
121 u32 tmp;
122 int k;
123
124
125
126
127 for (;; fentry++) {
128#if defined(CONFIG_X86_32)
129#define K (fentry->k)
130#else
131 const u32 K = fentry->k;
132#endif
133
134 switch (fentry->code) {
135 case BPF_S_ALU_ADD_X:
136 A += X;
137 continue;
138 case BPF_S_ALU_ADD_K:
139 A += K;
140 continue;
141 case BPF_S_ALU_SUB_X:
142 A -= X;
143 continue;
144 case BPF_S_ALU_SUB_K:
145 A -= K;
146 continue;
147 case BPF_S_ALU_MUL_X:
148 A *= X;
149 continue;
150 case BPF_S_ALU_MUL_K:
151 A *= K;
152 continue;
153 case BPF_S_ALU_DIV_X:
154 if (X == 0)
155 return 0;
156 A /= X;
157 continue;
158 case BPF_S_ALU_DIV_K:
159 A = reciprocal_divide(A, K);
160 continue;
161 case BPF_S_ALU_AND_X:
162 A &= X;
163 continue;
164 case BPF_S_ALU_AND_K:
165 A &= K;
166 continue;
167 case BPF_S_ALU_OR_X:
168 A |= X;
169 continue;
170 case BPF_S_ALU_OR_K:
171 A |= K;
172 continue;
173 case BPF_S_ALU_LSH_X:
174 A <<= X;
175 continue;
176 case BPF_S_ALU_LSH_K:
177 A <<= K;
178 continue;
179 case BPF_S_ALU_RSH_X:
180 A >>= X;
181 continue;
182 case BPF_S_ALU_RSH_K:
183 A >>= K;
184 continue;
185 case BPF_S_ALU_NEG:
186 A = -A;
187 continue;
188 case BPF_S_JMP_JA:
189 fentry += K;
190 continue;
191 case BPF_S_JMP_JGT_K:
192 fentry += (A > K) ? fentry->jt : fentry->jf;
193 continue;
194 case BPF_S_JMP_JGE_K:
195 fentry += (A >= K) ? fentry->jt : fentry->jf;
196 continue;
197 case BPF_S_JMP_JEQ_K:
198 fentry += (A == K) ? fentry->jt : fentry->jf;
199 continue;
200 case BPF_S_JMP_JSET_K:
201 fentry += (A & K) ? fentry->jt : fentry->jf;
202 continue;
203 case BPF_S_JMP_JGT_X:
204 fentry += (A > X) ? fentry->jt : fentry->jf;
205 continue;
206 case BPF_S_JMP_JGE_X:
207 fentry += (A >= X) ? fentry->jt : fentry->jf;
208 continue;
209 case BPF_S_JMP_JEQ_X:
210 fentry += (A == X) ? fentry->jt : fentry->jf;
211 continue;
212 case BPF_S_JMP_JSET_X:
213 fentry += (A & X) ? fentry->jt : fentry->jf;
214 continue;
215 case BPF_S_LD_W_ABS:
216 k = K;
217load_w:
218 ptr = load_pointer(skb, k, 4, &tmp);
219 if (ptr != NULL) {
220 A = get_unaligned_be32(ptr);
221 continue;
222 }
223 return 0;
224 case BPF_S_LD_H_ABS:
225 k = K;
226load_h:
227 ptr = load_pointer(skb, k, 2, &tmp);
228 if (ptr != NULL) {
229 A = get_unaligned_be16(ptr);
230 continue;
231 }
232 return 0;
233 case BPF_S_LD_B_ABS:
234 k = K;
235load_b:
236 ptr = load_pointer(skb, k, 1, &tmp);
237 if (ptr != NULL) {
238 A = *(u8 *)ptr;
239 continue;
240 }
241 return 0;
242 case BPF_S_LD_W_LEN:
243 A = skb->len;
244 continue;
245 case BPF_S_LDX_W_LEN:
246 X = skb->len;
247 continue;
248 case BPF_S_LD_W_IND:
249 k = X + K;
250 goto load_w;
251 case BPF_S_LD_H_IND:
252 k = X + K;
253 goto load_h;
254 case BPF_S_LD_B_IND:
255 k = X + K;
256 goto load_b;
257 case BPF_S_LDX_B_MSH:
258 ptr = load_pointer(skb, K, 1, &tmp);
259 if (ptr != NULL) {
260 X = (*(u8 *)ptr & 0xf) << 2;
261 continue;
262 }
263 return 0;
264 case BPF_S_LD_IMM:
265 A = K;
266 continue;
267 case BPF_S_LDX_IMM:
268 X = K;
269 continue;
270 case BPF_S_LD_MEM:
271 A = mem[K];
272 continue;
273 case BPF_S_LDX_MEM:
274 X = mem[K];
275 continue;
276 case BPF_S_MISC_TAX:
277 X = A;
278 continue;
279 case BPF_S_MISC_TXA:
280 A = X;
281 continue;
282 case BPF_S_RET_K:
283 return K;
284 case BPF_S_RET_A:
285 return A;
286 case BPF_S_ST:
287 mem[K] = A;
288 continue;
289 case BPF_S_STX:
290 mem[K] = X;
291 continue;
292 case BPF_S_ANC_PROTOCOL:
293 A = ntohs(skb->protocol);
294 continue;
295 case BPF_S_ANC_PKTTYPE:
296 A = skb->pkt_type;
297 continue;
298 case BPF_S_ANC_IFINDEX:
299 if (!skb->dev)
300 return 0;
301 A = skb->dev->ifindex;
302 continue;
303 case BPF_S_ANC_MARK:
304 A = skb->mark;
305 continue;
306 case BPF_S_ANC_QUEUE:
307 A = skb->queue_mapping;
308 continue;
309 case BPF_S_ANC_HATYPE:
310 if (!skb->dev)
311 return 0;
312 A = skb->dev->type;
313 continue;
314 case BPF_S_ANC_RXHASH:
315 A = skb->rxhash;
316 continue;
317 case BPF_S_ANC_CPU:
318 A = raw_smp_processor_id();
319 continue;
320 case BPF_S_ANC_NLATTR: {
321 struct nlattr *nla;
322
323 if (skb_is_nonlinear(skb))
324 return 0;
325 if (A > skb->len - sizeof(struct nlattr))
326 return 0;
327
328 nla = nla_find((struct nlattr *)&skb->data[A],
329 skb->len - A, X);
330 if (nla)
331 A = (void *)nla - (void *)skb->data;
332 else
333 A = 0;
334 continue;
335 }
336 case BPF_S_ANC_NLATTR_NEST: {
337 struct nlattr *nla;
338
339 if (skb_is_nonlinear(skb))
340 return 0;
341 if (A > skb->len - sizeof(struct nlattr))
342 return 0;
343
344 nla = (struct nlattr *)&skb->data[A];
345 if (nla->nla_len > A - skb->len)
346 return 0;
347
348 nla = nla_find_nested(nla, X);
349 if (nla)
350 A = (void *)nla - (void *)skb->data;
351 else
352 A = 0;
353 continue;
354 }
355 default:
356 WARN_RATELIMIT(1, "Unknown code:%u jt:%u tf:%u k:%u\n",
357 fentry->code, fentry->jt,
358 fentry->jf, fentry->k);
359 return 0;
360 }
361 }
362
363 return 0;
364}
365EXPORT_SYMBOL(sk_run_filter);
366
367
368
369
370
371
372
373
374
375
376static int check_load_and_stores(struct sock_filter *filter, int flen)
377{
378 u16 *masks, memvalid = 0;
379 int pc, ret = 0;
380
381 BUILD_BUG_ON(BPF_MEMWORDS > 16);
382 masks = kmalloc(flen * sizeof(*masks), GFP_KERNEL);
383 if (!masks)
384 return -ENOMEM;
385 memset(masks, 0xff, flen * sizeof(*masks));
386
387 for (pc = 0; pc < flen; pc++) {
388 memvalid &= masks[pc];
389
390 switch (filter[pc].code) {
391 case BPF_S_ST:
392 case BPF_S_STX:
393 memvalid |= (1 << filter[pc].k);
394 break;
395 case BPF_S_LD_MEM:
396 case BPF_S_LDX_MEM:
397 if (!(memvalid & (1 << filter[pc].k))) {
398 ret = -EINVAL;
399 goto error;
400 }
401 break;
402 case BPF_S_JMP_JA:
403
404 masks[pc + 1 + filter[pc].k] &= memvalid;
405 memvalid = ~0;
406 break;
407 case BPF_S_JMP_JEQ_K:
408 case BPF_S_JMP_JEQ_X:
409 case BPF_S_JMP_JGE_K:
410 case BPF_S_JMP_JGE_X:
411 case BPF_S_JMP_JGT_K:
412 case BPF_S_JMP_JGT_X:
413 case BPF_S_JMP_JSET_X:
414 case BPF_S_JMP_JSET_K:
415
416 masks[pc + 1 + filter[pc].jt] &= memvalid;
417 masks[pc + 1 + filter[pc].jf] &= memvalid;
418 memvalid = ~0;
419 break;
420 }
421 }
422error:
423 kfree(masks);
424 return ret;
425}
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441int sk_chk_filter(struct sock_filter *filter, unsigned int flen)
442{
443
444
445
446
447 static const u8 codes[] = {
448 [BPF_ALU|BPF_ADD|BPF_K] = BPF_S_ALU_ADD_K,
449 [BPF_ALU|BPF_ADD|BPF_X] = BPF_S_ALU_ADD_X,
450 [BPF_ALU|BPF_SUB|BPF_K] = BPF_S_ALU_SUB_K,
451 [BPF_ALU|BPF_SUB|BPF_X] = BPF_S_ALU_SUB_X,
452 [BPF_ALU|BPF_MUL|BPF_K] = BPF_S_ALU_MUL_K,
453 [BPF_ALU|BPF_MUL|BPF_X] = BPF_S_ALU_MUL_X,
454 [BPF_ALU|BPF_DIV|BPF_X] = BPF_S_ALU_DIV_X,
455 [BPF_ALU|BPF_AND|BPF_K] = BPF_S_ALU_AND_K,
456 [BPF_ALU|BPF_AND|BPF_X] = BPF_S_ALU_AND_X,
457 [BPF_ALU|BPF_OR|BPF_K] = BPF_S_ALU_OR_K,
458 [BPF_ALU|BPF_OR|BPF_X] = BPF_S_ALU_OR_X,
459 [BPF_ALU|BPF_LSH|BPF_K] = BPF_S_ALU_LSH_K,
460 [BPF_ALU|BPF_LSH|BPF_X] = BPF_S_ALU_LSH_X,
461 [BPF_ALU|BPF_RSH|BPF_K] = BPF_S_ALU_RSH_K,
462 [BPF_ALU|BPF_RSH|BPF_X] = BPF_S_ALU_RSH_X,
463 [BPF_ALU|BPF_NEG] = BPF_S_ALU_NEG,
464 [BPF_LD|BPF_W|BPF_ABS] = BPF_S_LD_W_ABS,
465 [BPF_LD|BPF_H|BPF_ABS] = BPF_S_LD_H_ABS,
466 [BPF_LD|BPF_B|BPF_ABS] = BPF_S_LD_B_ABS,
467 [BPF_LD|BPF_W|BPF_LEN] = BPF_S_LD_W_LEN,
468 [BPF_LD|BPF_W|BPF_IND] = BPF_S_LD_W_IND,
469 [BPF_LD|BPF_H|BPF_IND] = BPF_S_LD_H_IND,
470 [BPF_LD|BPF_B|BPF_IND] = BPF_S_LD_B_IND,
471 [BPF_LD|BPF_IMM] = BPF_S_LD_IMM,
472 [BPF_LDX|BPF_W|BPF_LEN] = BPF_S_LDX_W_LEN,
473 [BPF_LDX|BPF_B|BPF_MSH] = BPF_S_LDX_B_MSH,
474 [BPF_LDX|BPF_IMM] = BPF_S_LDX_IMM,
475 [BPF_MISC|BPF_TAX] = BPF_S_MISC_TAX,
476 [BPF_MISC|BPF_TXA] = BPF_S_MISC_TXA,
477 [BPF_RET|BPF_K] = BPF_S_RET_K,
478 [BPF_RET|BPF_A] = BPF_S_RET_A,
479 [BPF_ALU|BPF_DIV|BPF_K] = BPF_S_ALU_DIV_K,
480 [BPF_LD|BPF_MEM] = BPF_S_LD_MEM,
481 [BPF_LDX|BPF_MEM] = BPF_S_LDX_MEM,
482 [BPF_ST] = BPF_S_ST,
483 [BPF_STX] = BPF_S_STX,
484 [BPF_JMP|BPF_JA] = BPF_S_JMP_JA,
485 [BPF_JMP|BPF_JEQ|BPF_K] = BPF_S_JMP_JEQ_K,
486 [BPF_JMP|BPF_JEQ|BPF_X] = BPF_S_JMP_JEQ_X,
487 [BPF_JMP|BPF_JGE|BPF_K] = BPF_S_JMP_JGE_K,
488 [BPF_JMP|BPF_JGE|BPF_X] = BPF_S_JMP_JGE_X,
489 [BPF_JMP|BPF_JGT|BPF_K] = BPF_S_JMP_JGT_K,
490 [BPF_JMP|BPF_JGT|BPF_X] = BPF_S_JMP_JGT_X,
491 [BPF_JMP|BPF_JSET|BPF_K] = BPF_S_JMP_JSET_K,
492 [BPF_JMP|BPF_JSET|BPF_X] = BPF_S_JMP_JSET_X,
493 };
494 int pc;
495
496 if (flen == 0 || flen > BPF_MAXINSNS)
497 return -EINVAL;
498
499
500 for (pc = 0; pc < flen; pc++) {
501 struct sock_filter *ftest = &filter[pc];
502 u16 code = ftest->code;
503
504 if (code >= ARRAY_SIZE(codes))
505 return -EINVAL;
506 code = codes[code];
507 if (!code)
508 return -EINVAL;
509
510 switch (code) {
511 case BPF_S_ALU_DIV_K:
512
513 if (ftest->k == 0)
514 return -EINVAL;
515 ftest->k = reciprocal_value(ftest->k);
516 break;
517 case BPF_S_LD_MEM:
518 case BPF_S_LDX_MEM:
519 case BPF_S_ST:
520 case BPF_S_STX:
521
522 if (ftest->k >= BPF_MEMWORDS)
523 return -EINVAL;
524 break;
525 case BPF_S_JMP_JA:
526
527
528
529
530
531 if (ftest->k >= (unsigned)(flen-pc-1))
532 return -EINVAL;
533 break;
534 case BPF_S_JMP_JEQ_K:
535 case BPF_S_JMP_JEQ_X:
536 case BPF_S_JMP_JGE_K:
537 case BPF_S_JMP_JGE_X:
538 case BPF_S_JMP_JGT_K:
539 case BPF_S_JMP_JGT_X:
540 case BPF_S_JMP_JSET_X:
541 case BPF_S_JMP_JSET_K:
542
543 if (pc + ftest->jt + 1 >= flen ||
544 pc + ftest->jf + 1 >= flen)
545 return -EINVAL;
546 break;
547 case BPF_S_LD_W_ABS:
548 case BPF_S_LD_H_ABS:
549 case BPF_S_LD_B_ABS:
550#define ANCILLARY(CODE) case SKF_AD_OFF + SKF_AD_##CODE: \
551 code = BPF_S_ANC_##CODE; \
552 break
553 switch (ftest->k) {
554 ANCILLARY(PROTOCOL);
555 ANCILLARY(PKTTYPE);
556 ANCILLARY(IFINDEX);
557 ANCILLARY(NLATTR);
558 ANCILLARY(NLATTR_NEST);
559 ANCILLARY(MARK);
560 ANCILLARY(QUEUE);
561 ANCILLARY(HATYPE);
562 ANCILLARY(RXHASH);
563 ANCILLARY(CPU);
564 }
565 }
566 ftest->code = code;
567 }
568
569
570 switch (filter[flen - 1].code) {
571 case BPF_S_RET_K:
572 case BPF_S_RET_A:
573 return check_load_and_stores(filter, flen);
574 }
575 return -EINVAL;
576}
577EXPORT_SYMBOL(sk_chk_filter);
578
579
580
581
582
583void sk_filter_release_rcu(struct rcu_head *rcu)
584{
585 struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu);
586
587 bpf_jit_free(fp);
588 kfree(fp);
589}
590EXPORT_SYMBOL(sk_filter_release_rcu);
591
592
593
594
595
596
597
598
599
600
601
602int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
603{
604 struct sk_filter *fp, *old_fp;
605 unsigned int fsize = sizeof(struct sock_filter) * fprog->len;
606 int err;
607
608
609 if (fprog->filter == NULL)
610 return -EINVAL;
611
612 fp = sock_kmalloc(sk, fsize+sizeof(*fp), GFP_KERNEL);
613 if (!fp)
614 return -ENOMEM;
615 if (copy_from_user(fp->insns, fprog->filter, fsize)) {
616 sock_kfree_s(sk, fp, fsize+sizeof(*fp));
617 return -EFAULT;
618 }
619
620 atomic_set(&fp->refcnt, 1);
621 fp->len = fprog->len;
622 fp->bpf_func = sk_run_filter;
623
624 err = sk_chk_filter(fp->insns, fp->len);
625 if (err) {
626 sk_filter_uncharge(sk, fp);
627 return err;
628 }
629
630 bpf_jit_compile(fp);
631
632 old_fp = rcu_dereference_protected(sk->sk_filter,
633 sock_owned_by_user(sk));
634 rcu_assign_pointer(sk->sk_filter, fp);
635
636 if (old_fp)
637 sk_filter_uncharge(sk, old_fp);
638 return 0;
639}
640EXPORT_SYMBOL_GPL(sk_attach_filter);
641
642int sk_detach_filter(struct sock *sk)
643{
644 int ret = -ENOENT;
645 struct sk_filter *filter;
646
647 filter = rcu_dereference_protected(sk->sk_filter,
648 sock_owned_by_user(sk));
649 if (filter) {
650 RCU_INIT_POINTER(sk->sk_filter, NULL);
651 sk_filter_uncharge(sk, filter);
652 ret = 0;
653 }
654 return ret;
655}
656EXPORT_SYMBOL_GPL(sk_detach_filter);
657
