1
2
3
4
5
6
7
8
9
10#include <linux/moduleloader.h>
11#include <asm/cacheflush.h>
12#include <linux/netdevice.h>
13#include <linux/filter.h>
14#include <linux/if_vlan.h>
15
16
17
18
19
20
21
22
23
24
25
26
27
28int bpf_jit_enable __read_mostly;
29
30
31
32
33extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[];
34extern u8 sk_load_word_positive_offset[], sk_load_half_positive_offset[];
35extern u8 sk_load_byte_positive_offset[], sk_load_byte_msh_positive_offset[];
36extern u8 sk_load_word_negative_offset[], sk_load_half_negative_offset[];
37extern u8 sk_load_byte_negative_offset[], sk_load_byte_msh_negative_offset[];
38
39static inline u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
40{
41 if (len == 1)
42 *ptr = bytes;
43 else if (len == 2)
44 *(u16 *)ptr = bytes;
45 else {
46 *(u32 *)ptr = bytes;
47 barrier();
48 }
49 return ptr + len;
50}
51
52#define EMIT(bytes, len) do { prog = emit_code(prog, bytes, len); } while (0)
53
54#define EMIT1(b1) EMIT(b1, 1)
55#define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2)
56#define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
57#define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
58#define EMIT1_off32(b1, off) do { EMIT1(b1); EMIT(off, 4);} while (0)
59
60#define CLEAR_A() EMIT2(0x31, 0xc0)
61#define CLEAR_X() EMIT2(0x31, 0xdb)
62
63static inline bool is_imm8(int value)
64{
65 return value <= 127 && value >= -128;
66}
67
68static inline bool is_near(int offset)
69{
70 return offset <= 127 && offset >= -128;
71}
72
73#define EMIT_JMP(offset) \
74do { \
75 if (offset) { \
76 if (is_near(offset)) \
77 EMIT2(0xeb, offset); \
78 else \
79 EMIT1_off32(0xe9, offset); \
80 } \
81} while (0)
82
83
84
85
86#define X86_JB 0x72
87#define X86_JAE 0x73
88#define X86_JE 0x74
89#define X86_JNE 0x75
90#define X86_JBE 0x76
91#define X86_JA 0x77
92
93#define EMIT_COND_JMP(op, offset) \
94do { \
95 if (is_near(offset)) \
96 EMIT2(op, offset); \
97 else { \
98 EMIT2(0x0f, op + 0x10); \
99 EMIT(offset, 4); \
100 } \
101} while (0)
102
103#define COND_SEL(CODE, TOP, FOP) \
104 case CODE: \
105 t_op = TOP; \
106 f_op = FOP; \
107 goto cond_branch
108
109
110#define SEEN_DATAREF 1
111#define SEEN_XREG 2
112#define SEEN_MEM 4
113
114static inline void bpf_flush_icache(void *start, void *end)
115{
116 mm_segment_t old_fs = get_fs();
117
118 set_fs(KERNEL_DS);
119 smp_wmb();
120 flush_icache_range((unsigned long)start, (unsigned long)end);
121 set_fs(old_fs);
122}
123
124#define CHOOSE_LOAD_FUNC(K, func) \
125 ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
126
127
128
129
130#define PKT_TYPE_MAX 7
131static int pkt_type_offset(void)
132{
133 struct sk_buff skb_probe = {
134 .pkt_type = ~0,
135 };
136 char *ct = (char *)&skb_probe;
137 unsigned int off;
138
139 for (off = 0; off < sizeof(struct sk_buff); off++) {
140 if (ct[off] == PKT_TYPE_MAX)
141 return off;
142 }
143 pr_err_once("Please fix pkt_type_offset(), as pkt_type couldn't be found\n");
144 return -1;
145}
146
147void bpf_jit_compile(struct sk_filter *fp)
148{
149 u8 temp[64];
150 u8 *prog;
151 unsigned int proglen, oldproglen = 0;
152 int ilen, i;
153 int t_offset, f_offset;
154 u8 t_op, f_op, seen = 0, pass;
155 u8 *image = NULL;
156 u8 *func;
157 int pc_ret0 = -1;
158 unsigned int cleanup_addr;
159 unsigned int *addrs;
160 const struct sock_filter *filter = fp->insns;
161 int flen = fp->len;
162
163 if (!bpf_jit_enable)
164 return;
165
166 addrs = kmalloc(flen * sizeof(*addrs), GFP_KERNEL);
167 if (addrs == NULL)
168 return;
169
170
171
172
173 for (proglen = 0, i = 0; i < flen; i++) {
174 proglen += 64;
175 addrs[i] = proglen;
176 }
177 cleanup_addr = proglen;
178
179 for (pass = 0; pass < 10; pass++) {
180 u8 seen_or_pass0 = (pass == 0) ? (SEEN_XREG | SEEN_DATAREF | SEEN_MEM) : seen;
181
182 proglen = 0;
183 prog = temp;
184
185 if (seen_or_pass0) {
186 EMIT4(0x55, 0x48, 0x89, 0xe5);
187 EMIT4(0x48, 0x83, 0xec, 96);
188
189 if (seen_or_pass0 & (SEEN_XREG | SEEN_DATAREF))
190 EMIT4(0x48, 0x89, 0x5d, 0xf8);
191 if (seen_or_pass0 & SEEN_XREG)
192 CLEAR_X();
193
194
195
196
197
198
199
200 if (seen_or_pass0 & SEEN_DATAREF) {
201 if (offsetof(struct sk_buff, len) <= 127)
202
203 EMIT4(0x44, 0x8b, 0x4f, offsetof(struct sk_buff, len));
204 else {
205
206 EMIT3(0x44, 0x8b, 0x8f);
207 EMIT(offsetof(struct sk_buff, len), 4);
208 }
209 if (is_imm8(offsetof(struct sk_buff, data_len)))
210
211 EMIT4(0x44, 0x2b, 0x4f, offsetof(struct sk_buff, data_len));
212 else {
213 EMIT3(0x44, 0x2b, 0x8f);
214 EMIT(offsetof(struct sk_buff, data_len), 4);
215 }
216
217 if (is_imm8(offsetof(struct sk_buff, data)))
218
219 EMIT4(0x4c, 0x8b, 0x47, offsetof(struct sk_buff, data));
220 else {
221
222 EMIT3(0x4c, 0x8b, 0x87);
223 EMIT(offsetof(struct sk_buff, data), 4);
224 }
225 }
226 }
227
228 switch (filter[0].code) {
229 case BPF_S_RET_K:
230 case BPF_S_LD_W_LEN:
231 case BPF_S_ANC_PROTOCOL:
232 case BPF_S_ANC_IFINDEX:
233 case BPF_S_ANC_MARK:
234 case BPF_S_ANC_RXHASH:
235 case BPF_S_ANC_CPU:
236 case BPF_S_ANC_VLAN_TAG:
237 case BPF_S_ANC_VLAN_TAG_PRESENT:
238 case BPF_S_ANC_QUEUE:
239 case BPF_S_ANC_PKTTYPE:
240 case BPF_S_LD_W_ABS:
241 case BPF_S_LD_H_ABS:
242 case BPF_S_LD_B_ABS:
243
244 break;
245 default:
246
247 CLEAR_A();
248 }
249
250 for (i = 0; i < flen; i++) {
251 unsigned int K = filter[i].k;
252
253 switch (filter[i].code) {
254 case BPF_S_ALU_ADD_X:
255 seen |= SEEN_XREG;
256 EMIT2(0x01, 0xd8);
257 break;
258 case BPF_S_ALU_ADD_K:
259 if (!K)
260 break;
261 if (is_imm8(K))
262 EMIT3(0x83, 0xc0, K);
263 else
264 EMIT1_off32(0x05, K);
265 break;
266 case BPF_S_ALU_SUB_X:
267 seen |= SEEN_XREG;
268 EMIT2(0x29, 0xd8);
269 break;
270 case BPF_S_ALU_SUB_K:
271 if (!K)
272 break;
273 if (is_imm8(K))
274 EMIT3(0x83, 0xe8, K);
275 else
276 EMIT1_off32(0x2d, K);
277 break;
278 case BPF_S_ALU_MUL_X:
279 seen |= SEEN_XREG;
280 EMIT3(0x0f, 0xaf, 0xc3);
281 break;
282 case BPF_S_ALU_MUL_K:
283 if (is_imm8(K))
284 EMIT3(0x6b, 0xc0, K);
285 else {
286 EMIT2(0x69, 0xc0);
287 EMIT(K, 4);
288 }
289 break;
290 case BPF_S_ALU_DIV_X:
291 seen |= SEEN_XREG;
292 EMIT2(0x85, 0xdb);
293 if (pc_ret0 > 0) {
294
295
296
297
298 EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] -
299 (addrs[i] - 4));
300 } else {
301 EMIT_COND_JMP(X86_JNE, 2 + 5);
302 CLEAR_A();
303 EMIT1_off32(0xe9, cleanup_addr - (addrs[i] - 4));
304 }
305 EMIT4(0x31, 0xd2, 0xf7, 0xf3);
306 break;
307 case BPF_S_ALU_MOD_X:
308 seen |= SEEN_XREG;
309 EMIT2(0x85, 0xdb);
310 if (pc_ret0 > 0) {
311
312
313
314
315 EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] -
316 (addrs[i] - 6));
317 } else {
318 EMIT_COND_JMP(X86_JNE, 2 + 5);
319 CLEAR_A();
320 EMIT1_off32(0xe9, cleanup_addr - (addrs[i] - 6));
321 }
322 EMIT2(0x31, 0xd2);
323 EMIT2(0xf7, 0xf3);
324 EMIT2(0x89, 0xd0);
325 break;
326 case BPF_S_ALU_MOD_K:
327 EMIT2(0x31, 0xd2);
328 EMIT1(0xb9);EMIT(K, 4);
329 EMIT2(0xf7, 0xf1);
330 EMIT2(0x89, 0xd0);
331 break;
332 case BPF_S_ALU_DIV_K:
333 EMIT3(0x48, 0x69, 0xc0);
334 EMIT(K, 4);
335 EMIT4(0x48, 0xc1, 0xe8, 0x20);
336 break;
337 case BPF_S_ALU_AND_X:
338 seen |= SEEN_XREG;
339 EMIT2(0x21, 0xd8);
340 break;
341 case BPF_S_ALU_AND_K:
342 if (K >= 0xFFFFFF00) {
343 EMIT2(0x24, K & 0xFF);
344 } else if (K >= 0xFFFF0000) {
345 EMIT2(0x66, 0x25);
346 EMIT(K, 2);
347 } else {
348 EMIT1_off32(0x25, K);
349 }
350 break;
351 case BPF_S_ALU_OR_X:
352 seen |= SEEN_XREG;
353 EMIT2(0x09, 0xd8);
354 break;
355 case BPF_S_ALU_OR_K:
356 if (is_imm8(K))
357 EMIT3(0x83, 0xc8, K);
358 else
359 EMIT1_off32(0x0d, K);
360 break;
361 case BPF_S_ANC_ALU_XOR_X:
362 case BPF_S_ALU_XOR_X:
363 seen |= SEEN_XREG;
364 EMIT2(0x31, 0xd8);
365 break;
366 case BPF_S_ALU_XOR_K:
367 if (K == 0)
368 break;
369 if (is_imm8(K))
370 EMIT3(0x83, 0xf0, K);
371 else
372 EMIT1_off32(0x35, K);
373 break;
374 case BPF_S_ALU_LSH_X:
375 seen |= SEEN_XREG;
376 EMIT4(0x89, 0xd9, 0xd3, 0xe0);
377 break;
378 case BPF_S_ALU_LSH_K:
379 if (K == 0)
380 break;
381 else if (K == 1)
382 EMIT2(0xd1, 0xe0);
383 else
384 EMIT3(0xc1, 0xe0, K);
385 break;
386 case BPF_S_ALU_RSH_X:
387 seen |= SEEN_XREG;
388 EMIT4(0x89, 0xd9, 0xd3, 0xe8);
389 break;
390 case BPF_S_ALU_RSH_K:
391 if (K == 0)
392 break;
393 else if (K == 1)
394 EMIT2(0xd1, 0xe8);
395 else
396 EMIT3(0xc1, 0xe8, K);
397 break;
398 case BPF_S_ALU_NEG:
399 EMIT2(0xf7, 0xd8);
400 break;
401 case BPF_S_RET_K:
402 if (!K) {
403 if (pc_ret0 == -1)
404 pc_ret0 = i;
405 CLEAR_A();
406 } else {
407 EMIT1_off32(0xb8, K);
408 }
409
410 case BPF_S_RET_A:
411 if (seen_or_pass0) {
412 if (i != flen - 1) {
413 EMIT_JMP(cleanup_addr - addrs[i]);
414 break;
415 }
416 if (seen_or_pass0 & SEEN_XREG)
417 EMIT4(0x48, 0x8b, 0x5d, 0xf8);
418 EMIT1(0xc9);
419 }
420 EMIT1(0xc3);
421 break;
422 case BPF_S_MISC_TAX:
423 seen |= SEEN_XREG;
424 EMIT2(0x89, 0xc3);
425 break;
426 case BPF_S_MISC_TXA:
427 seen |= SEEN_XREG;
428 EMIT2(0x89, 0xd8);
429 break;
430 case BPF_S_LD_IMM:
431 if (!K)
432 CLEAR_A();
433 else
434 EMIT1_off32(0xb8, K);
435 break;
436 case BPF_S_LDX_IMM:
437 seen |= SEEN_XREG;
438 if (!K)
439 CLEAR_X();
440 else
441 EMIT1_off32(0xbb, K);
442 break;
443 case BPF_S_LD_MEM:
444 seen |= SEEN_MEM;
445 EMIT3(0x8b, 0x45, 0xf0 - K*4);
446 break;
447 case BPF_S_LDX_MEM:
448 seen |= SEEN_XREG | SEEN_MEM;
449 EMIT3(0x8b, 0x5d, 0xf0 - K*4);
450 break;
451 case BPF_S_ST:
452 seen |= SEEN_MEM;
453 EMIT3(0x89, 0x45, 0xf0 - K*4);
454 break;
455 case BPF_S_STX:
456 seen |= SEEN_XREG | SEEN_MEM;
457 EMIT3(0x89, 0x5d, 0xf0 - K*4);
458 break;
459 case BPF_S_LD_W_LEN:
460 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
461 if (is_imm8(offsetof(struct sk_buff, len)))
462
463 EMIT3(0x8b, 0x47, offsetof(struct sk_buff, len));
464 else {
465 EMIT2(0x8b, 0x87);
466 EMIT(offsetof(struct sk_buff, len), 4);
467 }
468 break;
469 case BPF_S_LDX_W_LEN:
470 seen |= SEEN_XREG;
471 if (is_imm8(offsetof(struct sk_buff, len)))
472
473 EMIT3(0x8b, 0x5f, offsetof(struct sk_buff, len));
474 else {
475 EMIT2(0x8b, 0x9f);
476 EMIT(offsetof(struct sk_buff, len), 4);
477 }
478 break;
479 case BPF_S_ANC_PROTOCOL:
480 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
481 if (is_imm8(offsetof(struct sk_buff, protocol))) {
482
483 EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, protocol));
484 } else {
485 EMIT3(0x0f, 0xb7, 0x87);
486 EMIT(offsetof(struct sk_buff, protocol), 4);
487 }
488 EMIT2(0x86, 0xc4);
489 break;
490 case BPF_S_ANC_IFINDEX:
491 if (is_imm8(offsetof(struct sk_buff, dev))) {
492
493 EMIT4(0x48, 0x8b, 0x47, offsetof(struct sk_buff, dev));
494 } else {
495 EMIT3(0x48, 0x8b, 0x87);
496 EMIT(offsetof(struct sk_buff, dev), 4);
497 }
498 EMIT3(0x48, 0x85, 0xc0);
499 EMIT_COND_JMP(X86_JE, cleanup_addr - (addrs[i] - 6));
500 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
501 EMIT2(0x8b, 0x80);
502 EMIT(offsetof(struct net_device, ifindex), 4);
503 break;
504 case BPF_S_ANC_MARK:
505 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
506 if (is_imm8(offsetof(struct sk_buff, mark))) {
507
508 EMIT3(0x8b, 0x47, offsetof(struct sk_buff, mark));
509 } else {
510 EMIT2(0x8b, 0x87);
511 EMIT(offsetof(struct sk_buff, mark), 4);
512 }
513 break;
514 case BPF_S_ANC_RXHASH:
515 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4);
516 if (is_imm8(offsetof(struct sk_buff, rxhash))) {
517
518 EMIT3(0x8b, 0x47, offsetof(struct sk_buff, rxhash));
519 } else {
520 EMIT2(0x8b, 0x87);
521 EMIT(offsetof(struct sk_buff, rxhash), 4);
522 }
523 break;
524 case BPF_S_ANC_QUEUE:
525 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
526 if (is_imm8(offsetof(struct sk_buff, queue_mapping))) {
527
528 EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, queue_mapping));
529 } else {
530 EMIT3(0x0f, 0xb7, 0x87);
531 EMIT(offsetof(struct sk_buff, queue_mapping), 4);
532 }
533 break;
534 case BPF_S_ANC_CPU:
535#ifdef CONFIG_SMP
536 EMIT4(0x65, 0x8b, 0x04, 0x25);
537 EMIT((u32)(unsigned long)&cpu_number, 4);
538#else
539 CLEAR_A();
540#endif
541 break;
542 case BPF_S_ANC_VLAN_TAG:
543 case BPF_S_ANC_VLAN_TAG_PRESENT:
544 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
545 if (is_imm8(offsetof(struct sk_buff, vlan_tci))) {
546
547 EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, vlan_tci));
548 } else {
549 EMIT3(0x0f, 0xb7, 0x87);
550 EMIT(offsetof(struct sk_buff, vlan_tci), 4);
551 }
552 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
553 if (filter[i].code == BPF_S_ANC_VLAN_TAG) {
554 EMIT3(0x80, 0xe4, 0xef);
555 } else {
556 EMIT3(0xc1, 0xe8, 0x0c);
557 EMIT3(0x83, 0xe0, 0x01);
558 }
559 break;
560 case BPF_S_ANC_PKTTYPE:
561 {
562 int off = pkt_type_offset();
563
564 if (off < 0)
565 goto out;
566 if (is_imm8(off)) {
567
568 EMIT4(0x0f, 0xb6, 0x47, off);
569 } else {
570
571 EMIT3(0x0f, 0xb6, 0x87);
572 EMIT(off, 4);
573 }
574 EMIT3(0x83, 0xe0, PKT_TYPE_MAX);
575 break;
576 }
577 case BPF_S_LD_W_ABS:
578 func = CHOOSE_LOAD_FUNC(K, sk_load_word);
579common_load: seen |= SEEN_DATAREF;
580 t_offset = func - (image + addrs[i]);
581 EMIT1_off32(0xbe, K);
582 EMIT1_off32(0xe8, t_offset);
583 break;
584 case BPF_S_LD_H_ABS:
585 func = CHOOSE_LOAD_FUNC(K, sk_load_half);
586 goto common_load;
587 case BPF_S_LD_B_ABS:
588 func = CHOOSE_LOAD_FUNC(K, sk_load_byte);
589 goto common_load;
590 case BPF_S_LDX_B_MSH:
591 func = CHOOSE_LOAD_FUNC(K, sk_load_byte_msh);
592 seen |= SEEN_DATAREF | SEEN_XREG;
593 t_offset = func - (image + addrs[i]);
594 EMIT1_off32(0xbe, K);
595 EMIT1_off32(0xe8, t_offset);
596 break;
597 case BPF_S_LD_W_IND:
598 func = sk_load_word;
599common_load_ind: seen |= SEEN_DATAREF | SEEN_XREG;
600 t_offset = func - (image + addrs[i]);
601 if (K) {
602 if (is_imm8(K)) {
603 EMIT3(0x8d, 0x73, K);
604 } else {
605 EMIT2(0x8d, 0xb3);
606 EMIT(K, 4);
607 }
608 } else {
609 EMIT2(0x89,0xde);
610 }
611 EMIT1_off32(0xe8, t_offset);
612 break;
613 case BPF_S_LD_H_IND:
614 func = sk_load_half;
615 goto common_load_ind;
616 case BPF_S_LD_B_IND:
617 func = sk_load_byte;
618 goto common_load_ind;
619 case BPF_S_JMP_JA:
620 t_offset = addrs[i + K] - addrs[i];
621 EMIT_JMP(t_offset);
622 break;
623 COND_SEL(BPF_S_JMP_JGT_K, X86_JA, X86_JBE);
624 COND_SEL(BPF_S_JMP_JGE_K, X86_JAE, X86_JB);
625 COND_SEL(BPF_S_JMP_JEQ_K, X86_JE, X86_JNE);
626 COND_SEL(BPF_S_JMP_JSET_K,X86_JNE, X86_JE);
627 COND_SEL(BPF_S_JMP_JGT_X, X86_JA, X86_JBE);
628 COND_SEL(BPF_S_JMP_JGE_X, X86_JAE, X86_JB);
629 COND_SEL(BPF_S_JMP_JEQ_X, X86_JE, X86_JNE);
630 COND_SEL(BPF_S_JMP_JSET_X,X86_JNE, X86_JE);
631
632cond_branch: f_offset = addrs[i + filter[i].jf] - addrs[i];
633 t_offset = addrs[i + filter[i].jt] - addrs[i];
634
635
636 if (filter[i].jt == filter[i].jf) {
637 EMIT_JMP(t_offset);
638 break;
639 }
640
641 switch (filter[i].code) {
642 case BPF_S_JMP_JGT_X:
643 case BPF_S_JMP_JGE_X:
644 case BPF_S_JMP_JEQ_X:
645 seen |= SEEN_XREG;
646 EMIT2(0x39, 0xd8);
647 break;
648 case BPF_S_JMP_JSET_X:
649 seen |= SEEN_XREG;
650 EMIT2(0x85, 0xd8);
651 break;
652 case BPF_S_JMP_JEQ_K:
653 if (K == 0) {
654 EMIT2(0x85, 0xc0);
655 break;
656 }
657 case BPF_S_JMP_JGT_K:
658 case BPF_S_JMP_JGE_K:
659 if (K <= 127)
660 EMIT3(0x83, 0xf8, K);
661 else
662 EMIT1_off32(0x3d, K);
663 break;
664 case BPF_S_JMP_JSET_K:
665 if (K <= 0xFF)
666 EMIT2(0xa8, K);
667 else if (!(K & 0xFFFF00FF))
668 EMIT3(0xf6, 0xc4, K >> 8);
669 else if (K <= 0xFFFF) {
670 EMIT2(0x66, 0xa9);
671 EMIT(K, 2);
672 } else {
673 EMIT1_off32(0xa9, K);
674 }
675 break;
676 }
677 if (filter[i].jt != 0) {
678 if (filter[i].jf && f_offset)
679 t_offset += is_near(f_offset) ? 2 : 5;
680 EMIT_COND_JMP(t_op, t_offset);
681 if (filter[i].jf)
682 EMIT_JMP(f_offset);
683 break;
684 }
685 EMIT_COND_JMP(f_op, f_offset);
686 break;
687 default:
688
689 goto out;
690 }
691 ilen = prog - temp;
692 if (image) {
693 if (unlikely(proglen + ilen > oldproglen)) {
694 pr_err("bpb_jit_compile fatal error\n");
695 kfree(addrs);
696 module_free(NULL, image);
697 return;
698 }
699 memcpy(image + proglen, temp, ilen);
700 }
701 proglen += ilen;
702 addrs[i] = proglen;
703 prog = temp;
704 }
705
706
707
708 cleanup_addr = proglen - 1;
709 if (seen_or_pass0)
710 cleanup_addr -= 1;
711 if (seen_or_pass0 & SEEN_XREG)
712 cleanup_addr -= 4;
713
714 if (image) {
715 if (proglen != oldproglen)
716 pr_err("bpb_jit_compile proglen=%u != oldproglen=%u\n", proglen, oldproglen);
717 break;
718 }
719 if (proglen == oldproglen) {
720 image = module_alloc(max_t(unsigned int,
721 proglen,
722 sizeof(struct work_struct)));
723 if (!image)
724 goto out;
725 }
726 oldproglen = proglen;
727 }
728
729 if (bpf_jit_enable > 1)
730 bpf_jit_dump(flen, proglen, pass, image);
731
732 if (image) {
733 bpf_flush_icache(image, image + proglen);
734 fp->bpf_func = (void *)image;
735 }
736out:
737 kfree(addrs);
738 return;
739}
740
741static void jit_free_defer(struct work_struct *arg)
742{
743 module_free(NULL, arg);
744}
745
746
747
748
749void bpf_jit_free(struct sk_filter *fp)
750{
751 if (fp->bpf_func != sk_run_filter) {
752 struct work_struct *work = (struct work_struct *)fp->bpf_func;
753
754 INIT_WORK(work, jit_free_defer);
755 schedule_work(work);
756 }
757}
758