1
2
3
4
5
6
7
8
9#include <bpf/libbpf.h>
10#include "perf.h"
11#include "debug.h"
12#include "bpf-loader.h"
13#include "bpf-prologue.h"
14#include "probe-finder.h"
15#include <dwarf-regs.h>
16#include <linux/filter.h>
17
18#define BPF_REG_SIZE 8
19
20#define JMP_TO_ERROR_CODE -1
21#define JMP_TO_SUCCESS_CODE -2
22#define JMP_TO_USER_CODE -3
23
24struct bpf_insn_pos {
25 struct bpf_insn *begin;
26 struct bpf_insn *end;
27 struct bpf_insn *pos;
28};
29
30static inline int
31pos_get_cnt(struct bpf_insn_pos *pos)
32{
33 return pos->pos - pos->begin;
34}
35
36static int
37append_insn(struct bpf_insn new_insn, struct bpf_insn_pos *pos)
38{
39 if (!pos->pos)
40 return -BPF_LOADER_ERRNO__PROLOGUE2BIG;
41
42 if (pos->pos + 1 >= pos->end) {
43 pr_err("bpf prologue: prologue too long\n");
44 pos->pos = NULL;
45 return -BPF_LOADER_ERRNO__PROLOGUE2BIG;
46 }
47
48 *(pos->pos)++ = new_insn;
49 return 0;
50}
51
52static int
53check_pos(struct bpf_insn_pos *pos)
54{
55 if (!pos->pos || pos->pos >= pos->end)
56 return -BPF_LOADER_ERRNO__PROLOGUE2BIG;
57 return 0;
58}
59
60
61#define ins(i, p) append_insn((i), (p))
62
63
64
65
66
67
68
69static int
70gen_ldx_reg_from_ctx(struct bpf_insn_pos *pos, int ctx_reg,
71 const char *reg, int target_reg)
72{
73 int offset = regs_query_register_offset(reg);
74
75 if (offset < 0) {
76 pr_err("bpf: prologue: failed to get register %s\n",
77 reg);
78 return offset;
79 }
80 ins(BPF_LDX_MEM(BPF_DW, target_reg, ctx_reg, offset), pos);
81
82 return check_pos(pos);
83}
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99static int
100gen_read_mem(struct bpf_insn_pos *pos,
101 int src_base_addr_reg,
102 int dst_addr_reg,
103 long offset)
104{
105
106 if (src_base_addr_reg != BPF_REG_ARG3)
107 ins(BPF_MOV64_REG(BPF_REG_ARG3, src_base_addr_reg), pos);
108
109 if (offset)
110 ins(BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG3, offset), pos);
111
112
113 ins(BPF_ALU64_IMM(BPF_MOV, BPF_REG_ARG2, BPF_REG_SIZE), pos);
114
115
116 if (dst_addr_reg != BPF_REG_ARG1)
117 ins(BPF_MOV64_REG(BPF_REG_ARG1, dst_addr_reg), pos);
118
119
120 ins(BPF_EMIT_CALL(BPF_FUNC_probe_read), pos);
121
122
123
124
125
126 ins(BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, JMP_TO_ERROR_CODE),
127 pos);
128
129 return check_pos(pos);
130}
131
132
133
134
135
136
137
138
139static int
140gen_prologue_fastpath(struct bpf_insn_pos *pos,
141 struct probe_trace_arg *args, int nargs)
142{
143 int i, err = 0;
144
145 for (i = 0; i < nargs; i++) {
146 err = gen_ldx_reg_from_ctx(pos, BPF_REG_1, args[i].value,
147 BPF_PROLOGUE_START_ARG_REG + i);
148 if (err)
149 goto errout;
150 }
151
152 return check_pos(pos);
153errout:
154 return err;
155}
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198static int
199gen_prologue_slowpath(struct bpf_insn_pos *pos,
200 struct probe_trace_arg *args, int nargs)
201{
202 int err, i;
203
204 for (i = 0; i < nargs; i++) {
205 struct probe_trace_arg *arg = &args[i];
206 const char *reg = arg->value;
207 struct probe_trace_arg_ref *ref = NULL;
208 int stack_offset = (i + 1) * -8;
209
210 pr_debug("prologue: fetch arg %d, base reg is %s\n",
211 i, reg);
212
213
214 err = gen_ldx_reg_from_ctx(pos, BPF_REG_CTX, reg,
215 BPF_REG_ARG3);
216 if (err) {
217 pr_err("prologue: failed to get offset of register %s\n",
218 reg);
219 goto errout;
220 }
221
222
223 ins(BPF_MOV64_REG(BPF_REG_7, BPF_REG_FP), pos);
224
225 ins(BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, stack_offset), pos);
226
227
228
229
230
231
232
233 ins(BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_ARG3,
234 stack_offset), pos);
235
236 ref = arg->ref;
237 while (ref) {
238 pr_debug("prologue: arg %d: offset %ld\n",
239 i, ref->offset);
240 err = gen_read_mem(pos, BPF_REG_3, BPF_REG_7,
241 ref->offset);
242 if (err) {
243 pr_err("prologue: failed to generate probe_read function call\n");
244 goto errout;
245 }
246
247 ref = ref->next;
248
249
250
251
252
253 if (ref)
254 ins(BPF_LDX_MEM(BPF_DW, BPF_REG_ARG3,
255 BPF_REG_FP, stack_offset), pos);
256 }
257 }
258
259
260 for (i = 0; i < nargs; i++)
261 ins(BPF_LDX_MEM(BPF_DW, BPF_PROLOGUE_START_ARG_REG + i,
262 BPF_REG_FP, -BPF_REG_SIZE * (i + 1)), pos);
263
264 ins(BPF_JMP_IMM(BPF_JA, BPF_REG_0, 0, JMP_TO_SUCCESS_CODE), pos);
265
266 return check_pos(pos);
267errout:
268 return err;
269}
270
271static int
272prologue_relocate(struct bpf_insn_pos *pos, struct bpf_insn *error_code,
273 struct bpf_insn *success_code, struct bpf_insn *user_code)
274{
275 struct bpf_insn *insn;
276
277 if (check_pos(pos))
278 return -BPF_LOADER_ERRNO__PROLOGUE2BIG;
279
280 for (insn = pos->begin; insn < pos->pos; insn++) {
281 struct bpf_insn *target;
282 u8 class = BPF_CLASS(insn->code);
283 u8 opcode;
284
285 if (class != BPF_JMP)
286 continue;
287 opcode = BPF_OP(insn->code);
288 if (opcode == BPF_CALL)
289 continue;
290
291 switch (insn->off) {
292 case JMP_TO_ERROR_CODE:
293 target = error_code;
294 break;
295 case JMP_TO_SUCCESS_CODE:
296 target = success_code;
297 break;
298 case JMP_TO_USER_CODE:
299 target = user_code;
300 break;
301 default:
302 pr_err("bpf prologue: internal error: relocation failed\n");
303 return -BPF_LOADER_ERRNO__PROLOGUE;
304 }
305
306 insn->off = target - (insn + 1);
307 }
308 return 0;
309}
310
311int bpf__gen_prologue(struct probe_trace_arg *args, int nargs,
312 struct bpf_insn *new_prog, size_t *new_cnt,
313 size_t cnt_space)
314{
315 struct bpf_insn *success_code = NULL;
316 struct bpf_insn *error_code = NULL;
317 struct bpf_insn *user_code = NULL;
318 struct bpf_insn_pos pos;
319 bool fastpath = true;
320 int err = 0, i;
321
322 if (!new_prog || !new_cnt)
323 return -EINVAL;
324
325 if (cnt_space > BPF_MAXINSNS)
326 cnt_space = BPF_MAXINSNS;
327
328 pos.begin = new_prog;
329 pos.end = new_prog + cnt_space;
330 pos.pos = new_prog;
331
332 if (!nargs) {
333 ins(BPF_ALU64_IMM(BPF_MOV, BPF_PROLOGUE_FETCH_RESULT_REG, 0),
334 &pos);
335
336 if (check_pos(&pos))
337 goto errout;
338
339 *new_cnt = pos_get_cnt(&pos);
340 return 0;
341 }
342
343 if (nargs > BPF_PROLOGUE_MAX_ARGS) {
344 pr_warning("bpf: prologue: %d arguments are dropped\n",
345 nargs - BPF_PROLOGUE_MAX_ARGS);
346 nargs = BPF_PROLOGUE_MAX_ARGS;
347 }
348
349
350 for (i = 0; i < nargs; i++) {
351 struct probe_trace_arg_ref *ref = args[i].ref;
352
353 if (args[i].value[0] == '@') {
354
355 pr_err("bpf: prologue: global %s%+ld not support\n",
356 args[i].value, ref ? ref->offset : 0);
357 return -ENOTSUP;
358 }
359
360 while (ref) {
361
362 fastpath = false;
363
364
365
366
367
368
369
370#ifdef __LP64__
371#define OFFSET_MAX ((1LL << 31) - 1)
372#define OFFSET_MIN ((1LL << 31) * -1)
373 if (ref->offset > OFFSET_MAX ||
374 ref->offset < OFFSET_MIN) {
375 pr_err("bpf: prologue: offset out of bound: %ld\n",
376 ref->offset);
377 return -BPF_LOADER_ERRNO__PROLOGUEOOB;
378 }
379#endif
380 ref = ref->next;
381 }
382 }
383 pr_debug("prologue: pass validation\n");
384
385 if (fastpath) {
386
387 pr_debug("prologue: fast path\n");
388 err = gen_prologue_fastpath(&pos, args, nargs);
389 if (err)
390 goto errout;
391 } else {
392 pr_debug("prologue: slow path\n");
393
394
395 ins(BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1), &pos);
396
397 err = gen_prologue_slowpath(&pos, args, nargs);
398 if (err)
399 goto errout;
400
401
402
403
404
405
406
407
408
409 error_code = pos.pos;
410 ins(BPF_ALU64_IMM(BPF_MOV, BPF_PROLOGUE_FETCH_RESULT_REG, 1),
411 &pos);
412
413 for (i = 0; i < nargs; i++)
414 ins(BPF_ALU64_IMM(BPF_MOV,
415 BPF_PROLOGUE_START_ARG_REG + i,
416 0),
417 &pos);
418 ins(BPF_JMP_IMM(BPF_JA, BPF_REG_0, 0, JMP_TO_USER_CODE),
419 &pos);
420 }
421
422
423
424
425
426
427 success_code = pos.pos;
428 ins(BPF_ALU64_IMM(BPF_MOV, BPF_PROLOGUE_FETCH_RESULT_REG, 0), &pos);
429
430
431
432
433
434 user_code = pos.pos;
435 if (!fastpath) {
436
437
438
439
440 ins(BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX), &pos);
441 err = prologue_relocate(&pos, error_code, success_code,
442 user_code);
443 if (err)
444 goto errout;
445 }
446
447 err = check_pos(&pos);
448 if (err)
449 goto errout;
450
451 *new_cnt = pos_get_cnt(&pos);
452 return 0;
453errout:
454 return err;
455}
456