1
2
3
4
5
6
7
8
9#include <linux/bpf.h>
10#include <linux/filter.h>
11#include "bpf_jit.h"
12
13
14#define NR_JIT_ITERATIONS 16
15
16static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
17{
18 const struct bpf_prog *prog = ctx->prog;
19 int i;
20
21 for (i = 0; i < prog->len; i++) {
22 const struct bpf_insn *insn = &prog->insnsi[i];
23 int ret;
24
25 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
26
27 if (ret > 0)
28 i++;
29 if (offset)
30 offset[i] = ctx->ninsns;
31 if (ret < 0)
32 return ret;
33 }
34 return 0;
35}
36
37bool bpf_jit_needs_zext(void)
38{
39 return true;
40}
41
42struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
43{
44 bool tmp_blinded = false, extra_pass = false;
45 struct bpf_prog *tmp, *orig_prog = prog;
46 int pass = 0, prev_ninsns = 0, i;
47 struct rv_jit_data *jit_data;
48 struct rv_jit_context *ctx;
49 unsigned int image_size = 0;
50
51 if (!prog->jit_requested)
52 return orig_prog;
53
54 tmp = bpf_jit_blind_constants(prog);
55 if (IS_ERR(tmp))
56 return orig_prog;
57 if (tmp != prog) {
58 tmp_blinded = true;
59 prog = tmp;
60 }
61
62 jit_data = prog->aux->jit_data;
63 if (!jit_data) {
64 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
65 if (!jit_data) {
66 prog = orig_prog;
67 goto out;
68 }
69 prog->aux->jit_data = jit_data;
70 }
71
72 ctx = &jit_data->ctx;
73
74 if (ctx->offset) {
75 extra_pass = true;
76 image_size = sizeof(u32) * ctx->ninsns;
77 goto skip_init_ctx;
78 }
79
80 ctx->prog = prog;
81 ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
82 if (!ctx->offset) {
83 prog = orig_prog;
84 goto out_offset;
85 }
86 for (i = 0; i < prog->len; i++) {
87 prev_ninsns += 32;
88 ctx->offset[i] = prev_ninsns;
89 }
90
91 for (i = 0; i < NR_JIT_ITERATIONS; i++) {
92 pass++;
93 ctx->ninsns = 0;
94 if (build_body(ctx, extra_pass, ctx->offset)) {
95 prog = orig_prog;
96 goto out_offset;
97 }
98 bpf_jit_build_prologue(ctx);
99 ctx->epilogue_offset = ctx->ninsns;
100 bpf_jit_build_epilogue(ctx);
101
102 if (ctx->ninsns == prev_ninsns) {
103 if (jit_data->header)
104 break;
105
106 image_size = sizeof(u32) * ctx->ninsns;
107 jit_data->header =
108 bpf_jit_binary_alloc(image_size,
109 &jit_data->image,
110 sizeof(u32),
111 bpf_fill_ill_insns);
112 if (!jit_data->header) {
113 prog = orig_prog;
114 goto out_offset;
115 }
116
117 ctx->insns = (u32 *)jit_data->image;
118
119
120
121
122 }
123 prev_ninsns = ctx->ninsns;
124 }
125
126 if (i == NR_JIT_ITERATIONS) {
127 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
128 bpf_jit_binary_free(jit_data->header);
129 prog = orig_prog;
130 goto out_offset;
131 }
132
133skip_init_ctx:
134 pass++;
135 ctx->ninsns = 0;
136
137 bpf_jit_build_prologue(ctx);
138 if (build_body(ctx, extra_pass, NULL)) {
139 bpf_jit_binary_free(jit_data->header);
140 prog = orig_prog;
141 goto out_offset;
142 }
143 bpf_jit_build_epilogue(ctx);
144
145 if (bpf_jit_enable > 1)
146 bpf_jit_dump(prog->len, image_size, pass, ctx->insns);
147
148 prog->bpf_func = (void *)ctx->insns;
149 prog->jited = 1;
150 prog->jited_len = image_size;
151
152 bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
153
154 if (!prog->is_func || extra_pass) {
155out_offset:
156 kfree(ctx->offset);
157 kfree(jit_data);
158 prog->aux->jit_data = NULL;
159 }
160out:
161
162 if (tmp_blinded)
163 bpf_jit_prog_release_other(prog, prog == orig_prog ?
164 tmp : orig_prog);
165 return prog;
166}
167