1
2
3
4
5
6
7
8
9
10
11
12#include <linux/moduleloader.h>
13#include <asm/cacheflush.h>
14#include <linux/netdevice.h>
15#include <linux/filter.h>
16#include <linux/if_vlan.h>
17
18#include "bpf_jit.h"
19
20#ifndef __BIG_ENDIAN
21
22#error "Little-endian PPC not supported in BPF compiler"
23#endif
24
25int bpf_jit_enable __read_mostly;
26
27
28static inline void bpf_flush_icache(void *start, void *end)
29{
30 smp_wmb();
31 flush_icache_range((unsigned long)start, (unsigned long)end);
32}
33
34static void bpf_jit_build_prologue(struct sk_filter *fp, u32 *image,
35 struct codegen_context *ctx)
36{
37 int i;
38 const struct sock_filter *filter = fp->insns;
39
40 if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) {
41
42 if (ctx->seen & SEEN_DATAREF) {
43
44 EMIT(PPC_INST_MFLR | __PPC_RT(R0));
45 PPC_STD(0, 1, 16);
46
47
48 PPC_STD(r_D, 1, -(8*(32-r_D)));
49 PPC_STD(r_HL, 1, -(8*(32-r_HL)));
50 }
51 if (ctx->seen & SEEN_MEM) {
52
53
54
55
56 for (i = r_M; i < (r_M+16); i++) {
57 if (ctx->seen & (1 << (i-r_M)))
58 PPC_STD(i, 1, -(8*(32-i)));
59 }
60 }
61 EMIT(PPC_INST_STDU | __PPC_RS(R1) | __PPC_RA(R1) |
62 (-BPF_PPC_STACKFRAME & 0xfffc));
63 }
64
65 if (ctx->seen & SEEN_DATAREF) {
66
67
68
69
70
71
72 PPC_LWZ_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
73 data_len));
74 PPC_LWZ_OFFS(r_HL, r_skb, offsetof(struct sk_buff, len));
75 PPC_SUB(r_HL, r_HL, r_scratch1);
76 PPC_LD_OFFS(r_D, r_skb, offsetof(struct sk_buff, data));
77 }
78
79 if (ctx->seen & SEEN_XREG) {
80
81
82
83
84 PPC_LI(r_X, 0);
85 }
86
87 switch (filter[0].code) {
88 case BPF_S_RET_K:
89 case BPF_S_LD_W_LEN:
90 case BPF_S_ANC_PROTOCOL:
91 case BPF_S_ANC_IFINDEX:
92 case BPF_S_ANC_MARK:
93 case BPF_S_ANC_RXHASH:
94 case BPF_S_ANC_VLAN_TAG:
95 case BPF_S_ANC_VLAN_TAG_PRESENT:
96 case BPF_S_ANC_CPU:
97 case BPF_S_ANC_QUEUE:
98 case BPF_S_LD_W_ABS:
99 case BPF_S_LD_H_ABS:
100 case BPF_S_LD_B_ABS:
101
102 break;
103 default:
104
105 PPC_LI(r_A, 0);
106 }
107}
108
109static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
110{
111 int i;
112
113 if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) {
114 PPC_ADDI(1, 1, BPF_PPC_STACKFRAME);
115 if (ctx->seen & SEEN_DATAREF) {
116 PPC_LD(0, 1, 16);
117 PPC_MTLR(0);
118 PPC_LD(r_D, 1, -(8*(32-r_D)));
119 PPC_LD(r_HL, 1, -(8*(32-r_HL)));
120 }
121 if (ctx->seen & SEEN_MEM) {
122
123 for (i = r_M; i < (r_M+16); i++) {
124 if (ctx->seen & (1 << (i-r_M)))
125 PPC_LD(i, 1, -(8*(32-i)));
126 }
127 }
128 }
129
130
131 PPC_BLR();
132}
133
134#define CHOOSE_LOAD_FUNC(K, func) \
135 ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
136
137
138static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
139 struct codegen_context *ctx,
140 unsigned int *addrs)
141{
142 const struct sock_filter *filter = fp->insns;
143 int flen = fp->len;
144 u8 *func;
145 unsigned int true_cond;
146 int i;
147
148
149 unsigned int exit_addr = addrs[flen];
150
151 for (i = 0; i < flen; i++) {
152 unsigned int K = filter[i].k;
153
154
155
156
157
158 addrs[i] = ctx->idx * 4;
159
160 switch (filter[i].code) {
161
162 case BPF_S_ALU_ADD_X:
163 ctx->seen |= SEEN_XREG;
164 PPC_ADD(r_A, r_A, r_X);
165 break;
166 case BPF_S_ALU_ADD_K:
167 if (!K)
168 break;
169 PPC_ADDI(r_A, r_A, IMM_L(K));
170 if (K >= 32768)
171 PPC_ADDIS(r_A, r_A, IMM_HA(K));
172 break;
173 case BPF_S_ALU_SUB_X:
174 ctx->seen |= SEEN_XREG;
175 PPC_SUB(r_A, r_A, r_X);
176 break;
177 case BPF_S_ALU_SUB_K:
178 if (!K)
179 break;
180 PPC_ADDI(r_A, r_A, IMM_L(-K));
181 if (K >= 32768)
182 PPC_ADDIS(r_A, r_A, IMM_HA(-K));
183 break;
184 case BPF_S_ALU_MUL_X:
185 ctx->seen |= SEEN_XREG;
186 PPC_MUL(r_A, r_A, r_X);
187 break;
188 case BPF_S_ALU_MUL_K:
189 if (K < 32768)
190 PPC_MULI(r_A, r_A, K);
191 else {
192 PPC_LI32(r_scratch1, K);
193 PPC_MUL(r_A, r_A, r_scratch1);
194 }
195 break;
196 case BPF_S_ALU_DIV_X:
197 ctx->seen |= SEEN_XREG;
198 PPC_CMPWI(r_X, 0);
199 if (ctx->pc_ret0 != -1) {
200 PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
201 } else {
202
203
204
205
206 PPC_BCC_SHORT(COND_NE, (ctx->idx*4)+12);
207 PPC_LI(r_ret, 0);
208 PPC_JMP(exit_addr);
209 }
210 PPC_DIVWU(r_A, r_A, r_X);
211 break;
212 case BPF_S_ALU_DIV_K:
213 PPC_LI32(r_scratch1, K);
214
215 PPC_MULHWU(r_A, r_A, r_scratch1);
216 break;
217 case BPF_S_ALU_AND_X:
218 ctx->seen |= SEEN_XREG;
219 PPC_AND(r_A, r_A, r_X);
220 break;
221 case BPF_S_ALU_AND_K:
222 if (!IMM_H(K))
223 PPC_ANDI(r_A, r_A, K);
224 else {
225 PPC_LI32(r_scratch1, K);
226 PPC_AND(r_A, r_A, r_scratch1);
227 }
228 break;
229 case BPF_S_ALU_OR_X:
230 ctx->seen |= SEEN_XREG;
231 PPC_OR(r_A, r_A, r_X);
232 break;
233 case BPF_S_ALU_OR_K:
234 if (IMM_L(K))
235 PPC_ORI(r_A, r_A, IMM_L(K));
236 if (K >= 65536)
237 PPC_ORIS(r_A, r_A, IMM_H(K));
238 break;
239 case BPF_S_ANC_ALU_XOR_X:
240 case BPF_S_ALU_XOR_X:
241 ctx->seen |= SEEN_XREG;
242 PPC_XOR(r_A, r_A, r_X);
243 break;
244 case BPF_S_ALU_XOR_K:
245 if (IMM_L(K))
246 PPC_XORI(r_A, r_A, IMM_L(K));
247 if (K >= 65536)
248 PPC_XORIS(r_A, r_A, IMM_H(K));
249 break;
250 case BPF_S_ALU_LSH_X:
251 ctx->seen |= SEEN_XREG;
252 PPC_SLW(r_A, r_A, r_X);
253 break;
254 case BPF_S_ALU_LSH_K:
255 if (K == 0)
256 break;
257 else
258 PPC_SLWI(r_A, r_A, K);
259 break;
260 case BPF_S_ALU_RSH_X:
261 ctx->seen |= SEEN_XREG;
262 PPC_SRW(r_A, r_A, r_X);
263 break;
264 case BPF_S_ALU_RSH_K:
265 if (K == 0)
266 break;
267 else
268 PPC_SRWI(r_A, r_A, K);
269 break;
270 case BPF_S_ALU_NEG:
271 PPC_NEG(r_A, r_A);
272 break;
273 case BPF_S_RET_K:
274 PPC_LI32(r_ret, K);
275 if (!K) {
276 if (ctx->pc_ret0 == -1)
277 ctx->pc_ret0 = i;
278 }
279
280
281
282
283
284
285
286 if (i != flen - 1) {
287
288
289
290
291
292
293
294 if (ctx->seen)
295 PPC_JMP(exit_addr);
296 else
297 PPC_BLR();
298 }
299 break;
300 case BPF_S_RET_A:
301 PPC_MR(r_ret, r_A);
302 if (i != flen - 1) {
303 if (ctx->seen)
304 PPC_JMP(exit_addr);
305 else
306 PPC_BLR();
307 }
308 break;
309 case BPF_S_MISC_TAX:
310 PPC_MR(r_X, r_A);
311 break;
312 case BPF_S_MISC_TXA:
313 ctx->seen |= SEEN_XREG;
314 PPC_MR(r_A, r_X);
315 break;
316
317
318 case BPF_S_LD_IMM:
319 PPC_LI32(r_A, K);
320 break;
321 case BPF_S_LDX_IMM:
322 PPC_LI32(r_X, K);
323 break;
324 case BPF_S_LD_MEM:
325 PPC_MR(r_A, r_M + (K & 0xf));
326 ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
327 break;
328 case BPF_S_LDX_MEM:
329 PPC_MR(r_X, r_M + (K & 0xf));
330 ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
331 break;
332 case BPF_S_ST:
333 PPC_MR(r_M + (K & 0xf), r_A);
334 ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
335 break;
336 case BPF_S_STX:
337 PPC_MR(r_M + (K & 0xf), r_X);
338 ctx->seen |= SEEN_XREG | SEEN_MEM | (1<<(K & 0xf));
339 break;
340 case BPF_S_LD_W_LEN:
341 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
342 PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, len));
343 break;
344 case BPF_S_LDX_W_LEN:
345 PPC_LWZ_OFFS(r_X, r_skb, offsetof(struct sk_buff, len));
346 break;
347
348
349
350
351
352
353
354
355 case BPF_S_ANC_PROTOCOL:
356 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
357 protocol) != 2);
358 PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
359 protocol));
360
361 break;
362 case BPF_S_ANC_IFINDEX:
363 PPC_LD_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
364 dev));
365 PPC_CMPDI(r_scratch1, 0);
366 if (ctx->pc_ret0 != -1) {
367 PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
368 } else {
369
370 PPC_BCC_SHORT(COND_NE, (ctx->idx*4)+12);
371 PPC_LI(r_ret, 0);
372 PPC_JMP(exit_addr);
373 }
374 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device,
375 ifindex) != 4);
376 PPC_LWZ_OFFS(r_A, r_scratch1,
377 offsetof(struct net_device, ifindex));
378 break;
379 case BPF_S_ANC_MARK:
380 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
381 PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
382 mark));
383 break;
384 case BPF_S_ANC_RXHASH:
385 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4);
386 PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
387 rxhash));
388 break;
389 case BPF_S_ANC_VLAN_TAG:
390 case BPF_S_ANC_VLAN_TAG_PRESENT:
391 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
392 PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
393 vlan_tci));
394 if (filter[i].code == BPF_S_ANC_VLAN_TAG)
395 PPC_ANDI(r_A, r_A, VLAN_VID_MASK);
396 else
397 PPC_ANDI(r_A, r_A, VLAN_TAG_PRESENT);
398 break;
399 case BPF_S_ANC_QUEUE:
400 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
401 queue_mapping) != 2);
402 PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
403 queue_mapping));
404 break;
405 case BPF_S_ANC_CPU:
406#ifdef CONFIG_SMP
407
408
409
410
411 BUILD_BUG_ON(FIELD_SIZEOF(struct paca_struct,
412 paca_index) != 2);
413 PPC_LHZ_OFFS(r_A, 13,
414 offsetof(struct paca_struct, paca_index));
415#else
416 PPC_LI(r_A, 0);
417#endif
418 break;
419
420
421 case BPF_S_LD_W_ABS:
422 func = CHOOSE_LOAD_FUNC(K, sk_load_word);
423 goto common_load;
424 case BPF_S_LD_H_ABS:
425 func = CHOOSE_LOAD_FUNC(K, sk_load_half);
426 goto common_load;
427 case BPF_S_LD_B_ABS:
428 func = CHOOSE_LOAD_FUNC(K, sk_load_byte);
429 common_load:
430
431 ctx->seen |= SEEN_DATAREF;
432 PPC_LI64(r_scratch1, func);
433 PPC_MTLR(r_scratch1);
434 PPC_LI32(r_addr, K);
435 PPC_BLRL();
436
437
438
439
440 PPC_BCC(COND_LT, exit_addr);
441 break;
442
443
444 case BPF_S_LD_W_IND:
445 func = sk_load_word;
446 goto common_load_ind;
447 case BPF_S_LD_H_IND:
448 func = sk_load_half;
449 goto common_load_ind;
450 case BPF_S_LD_B_IND:
451 func = sk_load_byte;
452 common_load_ind:
453
454
455
456
457 ctx->seen |= SEEN_DATAREF | SEEN_XREG;
458 PPC_LI64(r_scratch1, func);
459 PPC_MTLR(r_scratch1);
460 PPC_ADDI(r_addr, r_X, IMM_L(K));
461 if (K >= 32768)
462 PPC_ADDIS(r_addr, r_addr, IMM_HA(K));
463 PPC_BLRL();
464
465 PPC_BCC(COND_LT, exit_addr);
466 break;
467
468 case BPF_S_LDX_B_MSH:
469 func = CHOOSE_LOAD_FUNC(K, sk_load_byte_msh);
470 goto common_load;
471 break;
472
473
474 case BPF_S_JMP_JA:
475 if (K != 0)
476 PPC_JMP(addrs[i + 1 + K]);
477 break;
478
479 case BPF_S_JMP_JGT_K:
480 case BPF_S_JMP_JGT_X:
481 true_cond = COND_GT;
482 goto cond_branch;
483 case BPF_S_JMP_JGE_K:
484 case BPF_S_JMP_JGE_X:
485 true_cond = COND_GE;
486 goto cond_branch;
487 case BPF_S_JMP_JEQ_K:
488 case BPF_S_JMP_JEQ_X:
489 true_cond = COND_EQ;
490 goto cond_branch;
491 case BPF_S_JMP_JSET_K:
492 case BPF_S_JMP_JSET_X:
493 true_cond = COND_NE;
494
495 cond_branch:
496
497 if (filter[i].jt == filter[i].jf) {
498 if (filter[i].jt > 0)
499 PPC_JMP(addrs[i + 1 + filter[i].jt]);
500 break;
501 }
502
503 switch (filter[i].code) {
504 case BPF_S_JMP_JGT_X:
505 case BPF_S_JMP_JGE_X:
506 case BPF_S_JMP_JEQ_X:
507 ctx->seen |= SEEN_XREG;
508 PPC_CMPLW(r_A, r_X);
509 break;
510 case BPF_S_JMP_JSET_X:
511 ctx->seen |= SEEN_XREG;
512 PPC_AND_DOT(r_scratch1, r_A, r_X);
513 break;
514 case BPF_S_JMP_JEQ_K:
515 case BPF_S_JMP_JGT_K:
516 case BPF_S_JMP_JGE_K:
517 if (K < 32768)
518 PPC_CMPLWI(r_A, K);
519 else {
520 PPC_LI32(r_scratch1, K);
521 PPC_CMPLW(r_A, r_scratch1);
522 }
523 break;
524 case BPF_S_JMP_JSET_K:
525 if (K < 32768)
526
527 PPC_ANDI(r_scratch1, r_A, K);
528 else {
529 PPC_LI32(r_scratch1, K);
530 PPC_AND_DOT(r_scratch1, r_A,
531 r_scratch1);
532 }
533 break;
534 }
535
536
537
538
539 if (filter[i].jt == 0)
540
541 PPC_BCC(true_cond ^ COND_CMP_TRUE,
542 addrs[i + 1 + filter[i].jf]);
543 else {
544 PPC_BCC(true_cond, addrs[i + 1 + filter[i].jt]);
545 if (filter[i].jf != 0)
546 PPC_JMP(addrs[i + 1 + filter[i].jf]);
547 }
548 break;
549 default:
550
551
552
553
554 if (printk_ratelimit())
555 pr_err("BPF filter opcode %04x (@%d) unsupported\n",
556 filter[i].code, i);
557 return -ENOTSUPP;
558 }
559
560 }
561
562 addrs[i] = ctx->idx * 4;
563
564 return 0;
565}
566
567void bpf_jit_compile(struct sk_filter *fp)
568{
569 unsigned int proglen;
570 unsigned int alloclen;
571 u32 *image = NULL;
572 u32 *code_base;
573 unsigned int *addrs;
574 struct codegen_context cgctx;
575 int pass;
576 int flen = fp->len;
577
578 if (!bpf_jit_enable)
579 return;
580
581 addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
582 if (addrs == NULL)
583 return;
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635 cgctx.idx = 0;
636 cgctx.seen = 0;
637 cgctx.pc_ret0 = -1;
638
639 if (bpf_jit_build_body(fp, 0, &cgctx, addrs))
640
641 goto out;
642
643
644
645
646
647
648 bpf_jit_build_prologue(fp, 0, &cgctx);
649 bpf_jit_build_epilogue(0, &cgctx);
650
651 proglen = cgctx.idx * 4;
652 alloclen = proglen + FUNCTION_DESCR_SIZE;
653 image = module_alloc(alloclen);
654 if (!image)
655 goto out;
656
657 code_base = image + (FUNCTION_DESCR_SIZE/4);
658
659
660 for (pass = 1; pass < 3; pass++) {
661
662 cgctx.idx = 0;
663 bpf_jit_build_prologue(fp, code_base, &cgctx);
664 bpf_jit_build_body(fp, code_base, &cgctx, addrs);
665 bpf_jit_build_epilogue(code_base, &cgctx);
666
667 if (bpf_jit_enable > 1)
668 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
669 proglen - (cgctx.idx * 4), cgctx.seen);
670 }
671
672 if (bpf_jit_enable > 1)
673
674
675
676 bpf_jit_dump(flen, proglen, pass, code_base);
677
678 if (image) {
679 bpf_flush_icache(code_base, code_base + (proglen/4));
680
681 ((u64 *)image)[0] = (u64)code_base;
682 ((u64 *)image)[1] = local_paca->kernel_toc;
683 fp->bpf_func = (void *)image;
684 }
685out:
686 kfree(addrs);
687 return;
688}
689
690void bpf_jit_free(struct sk_filter *fp)
691{
692 if (fp->bpf_func != sk_run_filter)
693 module_free(NULL, fp->bpf_func);
694 kfree(fp);
695}
696