1
2
3
4
5
6
7
8
9
10#include <linux/moduleloader.h>
11#include <asm/cacheflush.h>
12#include <asm/asm-compat.h>
13#include <linux/netdevice.h>
14#include <linux/filter.h>
15#include <linux/if_vlan.h>
16#include <asm/kprobes.h>
17#include <linux/bpf.h>
18
19#include "bpf_jit.h"
20
21
22
23
24
25
26
27
28
29
30
31
32#define BPF_PPC_STACK_SAVE (15 * 4 + 4)
33
34#define BPF_PPC_STACKFRAME(ctx) (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size)
35
36#define PPC_EX32(r, i) EMIT(PPC_RAW_LI((r), (i) < 0 ? -1 : 0))
37
38
39#define BPF_PPC_NVR_MIN _R17
40#define BPF_PPC_TC _R16
41
42
43#define TMP_REG (MAX_BPF_JIT_REG + 0)
44
45
46void bpf_jit_init_reg_mapping(struct codegen_context *ctx)
47{
48
49 ctx->b2p[BPF_REG_0] = _R12;
50
51 ctx->b2p[BPF_REG_1] = _R4;
52 ctx->b2p[BPF_REG_2] = _R6;
53 ctx->b2p[BPF_REG_3] = _R8;
54 ctx->b2p[BPF_REG_4] = _R10;
55 ctx->b2p[BPF_REG_5] = _R22;
56
57 ctx->b2p[BPF_REG_6] = _R24;
58 ctx->b2p[BPF_REG_7] = _R26;
59 ctx->b2p[BPF_REG_8] = _R28;
60 ctx->b2p[BPF_REG_9] = _R30;
61
62 ctx->b2p[BPF_REG_FP] = _R18;
63
64 ctx->b2p[BPF_REG_AX] = _R20;
65 ctx->b2p[TMP_REG] = _R31;
66}
67
68static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
69{
70 if ((reg >= BPF_PPC_NVR_MIN && reg < 32) || reg == BPF_PPC_TC)
71 return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg);
72
73 WARN(true, "BPF JIT is asking about unknown registers, will crash the stack");
74
75 return BPF_PPC_STACKFRAME(ctx) - 4;
76}
77
78#define SEEN_VREG_MASK 0x1ff80000
79#define SEEN_NVREG_FULL_MASK 0x0003ffff
80#define SEEN_NVREG_TEMP_MASK 0x00001e01
81
82void bpf_jit_realloc_regs(struct codegen_context *ctx)
83{
84 unsigned int nvreg_mask;
85
86 if (ctx->seen & SEEN_FUNC)
87 nvreg_mask = SEEN_NVREG_TEMP_MASK;
88 else
89 nvreg_mask = SEEN_NVREG_FULL_MASK;
90
91 while (ctx->seen & nvreg_mask &&
92 (ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) {
93 int old = 32 - fls(ctx->seen & (nvreg_mask & 0xaaaaaaab));
94 int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa));
95 int i;
96
97 for (i = BPF_REG_0; i <= TMP_REG; i++) {
98 if (ctx->b2p[i] != old)
99 continue;
100 ctx->b2p[i] = new;
101 bpf_set_seen_register(ctx, new);
102 bpf_clear_seen_register(ctx, old);
103 if (i != TMP_REG) {
104 bpf_set_seen_register(ctx, new - 1);
105 bpf_clear_seen_register(ctx, old - 1);
106 }
107 break;
108 }
109 }
110}
111
112void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
113{
114 int i;
115
116
117 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_1), _R3));
118 EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_1) - 1, 0));
119 EMIT(PPC_RAW_STWU(_R1, _R1, -BPF_PPC_STACKFRAME(ctx)));
120
121
122
123
124
125
126 if (ctx->seen & SEEN_TAILCALL)
127 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_1) - 1, _R1,
128 bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
129 else
130 EMIT(PPC_RAW_NOP());
131
132#define BPF_TAILCALL_PROLOGUE_SIZE 16
133
134
135
136
137
138 if (ctx->seen & SEEN_FUNC)
139 EMIT(PPC_RAW_MFLR(_R0));
140
141
142
143
144 for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
145 if (bpf_is_seen_register(ctx, i))
146 EMIT(PPC_RAW_STW(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
147
148
149 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) {
150 EMIT(PPC_RAW_LWZ(bpf_to_ppc(BPF_REG_5) - 1, _R1, BPF_PPC_STACKFRAME(ctx)) + 8);
151 EMIT(PPC_RAW_LWZ(bpf_to_ppc(BPF_REG_5), _R1, BPF_PPC_STACKFRAME(ctx)) + 12);
152 }
153
154
155 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) {
156 EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_FP) - 1, 0));
157 EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1,
158 STACK_FRAME_MIN_SIZE + ctx->stack_size));
159 }
160
161 if (ctx->seen & SEEN_FUNC)
162 EMIT(PPC_RAW_STW(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
163}
164
165static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
166{
167 int i;
168
169
170 for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
171 if (bpf_is_seen_register(ctx, i))
172 EMIT(PPC_RAW_LWZ(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
173}
174
175void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
176{
177 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0)));
178
179 bpf_jit_emit_common_epilogue(image, ctx);
180
181
182
183 if (ctx->seen & SEEN_FUNC)
184 EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
185
186 EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME(ctx)));
187
188 if (ctx->seen & SEEN_FUNC)
189 EMIT(PPC_RAW_MTLR(_R0));
190
191 EMIT(PPC_RAW_BLR());
192}
193
194int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func)
195{
196 s32 rel = (s32)func - (s32)(image + ctx->idx);
197
198 if (image && rel < 0x2000000 && rel >= -0x2000000) {
199 PPC_BL(func);
200 EMIT(PPC_RAW_NOP());
201 EMIT(PPC_RAW_NOP());
202 EMIT(PPC_RAW_NOP());
203 } else {
204
205 EMIT(PPC_RAW_LIS(_R0, IMM_H(func)));
206 EMIT(PPC_RAW_ORI(_R0, _R0, IMM_L(func)));
207 EMIT(PPC_RAW_MTCTR(_R0));
208 EMIT(PPC_RAW_BCTRL());
209 }
210
211 return 0;
212}
213
214static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
215{
216
217
218
219
220
221
222 int b2p_bpf_array = bpf_to_ppc(BPF_REG_2);
223 int b2p_index = bpf_to_ppc(BPF_REG_3);
224
225
226
227
228
229 EMIT(PPC_RAW_LWZ(_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
230 EMIT(PPC_RAW_CMPLW(b2p_index, _R0));
231 EMIT(PPC_RAW_LWZ(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
232 PPC_BCC_SHORT(COND_GE, out);
233
234
235
236
237
238 EMIT(PPC_RAW_CMPLWI(_R0, MAX_TAIL_CALL_CNT));
239
240 EMIT(PPC_RAW_ADDIC(_R0, _R0, 1));
241 PPC_BCC_SHORT(COND_GE, out);
242
243
244 EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29));
245 EMIT(PPC_RAW_ADD(_R3, _R3, b2p_bpf_array));
246 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_array, ptrs)));
247 EMIT(PPC_RAW_STW(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
248
249
250
251
252
253 EMIT(PPC_RAW_CMPLWI(_R3, 0));
254 PPC_BCC_SHORT(COND_EQ, out);
255
256
257 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_prog, bpf_func)));
258
259 if (ctx->seen & SEEN_FUNC)
260 EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
261
262 EMIT(PPC_RAW_ADDIC(_R3, _R3, BPF_TAILCALL_PROLOGUE_SIZE));
263
264 if (ctx->seen & SEEN_FUNC)
265 EMIT(PPC_RAW_MTLR(_R0));
266
267 EMIT(PPC_RAW_MTCTR(_R3));
268
269 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_1)));
270
271
272 bpf_jit_emit_common_epilogue(image, ctx);
273
274 EMIT(PPC_RAW_BCTR());
275
276
277 return 0;
278}
279
280
281int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
282 u32 *addrs, int pass)
283{
284 const struct bpf_insn *insn = fp->insnsi;
285 int flen = fp->len;
286 int i, ret;
287
288
289 u32 exit_addr = addrs[flen];
290
291 for (i = 0; i < flen; i++) {
292 u32 code = insn[i].code;
293 u32 dst_reg = bpf_to_ppc(insn[i].dst_reg);
294 u32 dst_reg_h = dst_reg - 1;
295 u32 src_reg = bpf_to_ppc(insn[i].src_reg);
296 u32 src_reg_h = src_reg - 1;
297 u32 tmp_reg = bpf_to_ppc(TMP_REG);
298 u32 size = BPF_SIZE(code);
299 s16 off = insn[i].off;
300 s32 imm = insn[i].imm;
301 bool func_addr_fixed;
302 u64 func_addr;
303 u32 true_cond;
304 u32 tmp_idx;
305 int j;
306
307
308
309
310
311 addrs[i] = ctx->idx * 4;
312
313
314
315
316
317
318
319
320
321
322
323
324 if (dst_reg >= 3 && dst_reg < 32) {
325 bpf_set_seen_register(ctx, dst_reg);
326 bpf_set_seen_register(ctx, dst_reg_h);
327 }
328
329 if (src_reg >= 3 && src_reg < 32) {
330 bpf_set_seen_register(ctx, src_reg);
331 bpf_set_seen_register(ctx, src_reg_h);
332 }
333
334 switch (code) {
335
336
337
338 case BPF_ALU | BPF_ADD | BPF_X:
339 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
340 break;
341 case BPF_ALU64 | BPF_ADD | BPF_X:
342 EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, src_reg));
343 EMIT(PPC_RAW_ADDE(dst_reg_h, dst_reg_h, src_reg_h));
344 break;
345 case BPF_ALU | BPF_SUB | BPF_X:
346 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
347 break;
348 case BPF_ALU64 | BPF_SUB | BPF_X:
349 EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, dst_reg));
350 EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, dst_reg_h));
351 break;
352 case BPF_ALU | BPF_SUB | BPF_K:
353 imm = -imm;
354 fallthrough;
355 case BPF_ALU | BPF_ADD | BPF_K:
356 if (IMM_HA(imm) & 0xffff)
357 EMIT(PPC_RAW_ADDIS(dst_reg, dst_reg, IMM_HA(imm)));
358 if (IMM_L(imm))
359 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
360 break;
361 case BPF_ALU64 | BPF_SUB | BPF_K:
362 imm = -imm;
363 fallthrough;
364 case BPF_ALU64 | BPF_ADD | BPF_K:
365 if (!imm)
366 break;
367
368 if (imm >= -32768 && imm < 32768) {
369 EMIT(PPC_RAW_ADDIC(dst_reg, dst_reg, imm));
370 } else {
371 PPC_LI32(_R0, imm);
372 EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, _R0));
373 }
374 if (imm >= 0 || (BPF_OP(code) == BPF_SUB && imm == 0x80000000))
375 EMIT(PPC_RAW_ADDZE(dst_reg_h, dst_reg_h));
376 else
377 EMIT(PPC_RAW_ADDME(dst_reg_h, dst_reg_h));
378 break;
379 case BPF_ALU64 | BPF_MUL | BPF_X:
380 bpf_set_seen_register(ctx, tmp_reg);
381 EMIT(PPC_RAW_MULW(_R0, dst_reg, src_reg_h));
382 EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, src_reg));
383 EMIT(PPC_RAW_MULHWU(tmp_reg, dst_reg, src_reg));
384 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
385 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
386 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg));
387 break;
388 case BPF_ALU | BPF_MUL | BPF_X:
389 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
390 break;
391 case BPF_ALU | BPF_MUL | BPF_K:
392 if (imm >= -32768 && imm < 32768) {
393 EMIT(PPC_RAW_MULI(dst_reg, dst_reg, imm));
394 } else {
395 PPC_LI32(_R0, imm);
396 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, _R0));
397 }
398 break;
399 case BPF_ALU64 | BPF_MUL | BPF_K:
400 if (!imm) {
401 PPC_LI32(dst_reg, 0);
402 PPC_LI32(dst_reg_h, 0);
403 break;
404 }
405 if (imm == 1)
406 break;
407 if (imm == -1) {
408 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
409 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
410 break;
411 }
412 bpf_set_seen_register(ctx, tmp_reg);
413 PPC_LI32(tmp_reg, imm);
414 EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, tmp_reg));
415 if (imm < 0)
416 EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, dst_reg));
417 EMIT(PPC_RAW_MULHWU(_R0, dst_reg, tmp_reg));
418 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp_reg));
419 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
420 break;
421 case BPF_ALU | BPF_DIV | BPF_X:
422 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
423 break;
424 case BPF_ALU | BPF_MOD | BPF_X:
425 EMIT(PPC_RAW_DIVWU(_R0, dst_reg, src_reg));
426 EMIT(PPC_RAW_MULW(_R0, src_reg, _R0));
427 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0));
428 break;
429 case BPF_ALU64 | BPF_DIV | BPF_X:
430 return -EOPNOTSUPP;
431 case BPF_ALU64 | BPF_MOD | BPF_X:
432 return -EOPNOTSUPP;
433 case BPF_ALU | BPF_DIV | BPF_K:
434 if (!imm)
435 return -EINVAL;
436 if (imm == 1)
437 break;
438
439 PPC_LI32(_R0, imm);
440 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, _R0));
441 break;
442 case BPF_ALU | BPF_MOD | BPF_K:
443 if (!imm)
444 return -EINVAL;
445
446 if (!is_power_of_2((u32)imm)) {
447 bpf_set_seen_register(ctx, tmp_reg);
448 PPC_LI32(tmp_reg, imm);
449 EMIT(PPC_RAW_DIVWU(_R0, dst_reg, tmp_reg));
450 EMIT(PPC_RAW_MULW(_R0, tmp_reg, _R0));
451 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0));
452 break;
453 }
454 if (imm == 1)
455 EMIT(PPC_RAW_LI(dst_reg, 0));
456 else
457 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2((u32)imm), 31));
458
459 break;
460 case BPF_ALU64 | BPF_MOD | BPF_K:
461 if (!imm)
462 return -EINVAL;
463 if (imm < 0)
464 imm = -imm;
465 if (!is_power_of_2(imm))
466 return -EOPNOTSUPP;
467 if (imm == 1)
468 EMIT(PPC_RAW_LI(dst_reg, 0));
469 else
470 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2(imm), 31));
471 EMIT(PPC_RAW_LI(dst_reg_h, 0));
472 break;
473 case BPF_ALU64 | BPF_DIV | BPF_K:
474 if (!imm)
475 return -EINVAL;
476 if (!is_power_of_2(abs(imm)))
477 return -EOPNOTSUPP;
478
479 if (imm < 0) {
480 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
481 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
482 imm = -imm;
483 }
484 if (imm == 1)
485 break;
486 imm = ilog2(imm);
487 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
488 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
489 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
490 break;
491 case BPF_ALU | BPF_NEG:
492 EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
493 break;
494 case BPF_ALU64 | BPF_NEG:
495 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
496 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
497 break;
498
499
500
501
502 case BPF_ALU64 | BPF_AND | BPF_X:
503 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
504 EMIT(PPC_RAW_AND(dst_reg_h, dst_reg_h, src_reg_h));
505 break;
506 case BPF_ALU | BPF_AND | BPF_X:
507 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
508 break;
509 case BPF_ALU64 | BPF_AND | BPF_K:
510 if (imm >= 0)
511 EMIT(PPC_RAW_LI(dst_reg_h, 0));
512 fallthrough;
513 case BPF_ALU | BPF_AND | BPF_K:
514 if (!IMM_H(imm)) {
515 EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
516 } else if (!IMM_L(imm)) {
517 EMIT(PPC_RAW_ANDIS(dst_reg, dst_reg, IMM_H(imm)));
518 } else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) {
519 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0,
520 32 - fls(imm), 32 - ffs(imm)));
521 } else {
522 PPC_LI32(_R0, imm);
523 EMIT(PPC_RAW_AND(dst_reg, dst_reg, _R0));
524 }
525 break;
526 case BPF_ALU64 | BPF_OR | BPF_X:
527 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
528 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, src_reg_h));
529 break;
530 case BPF_ALU | BPF_OR | BPF_X:
531 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
532 break;
533 case BPF_ALU64 | BPF_OR | BPF_K:
534
535 if (imm < 0)
536 EMIT(PPC_RAW_LI(dst_reg_h, -1));
537 fallthrough;
538 case BPF_ALU | BPF_OR | BPF_K:
539 if (IMM_L(imm))
540 EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
541 if (IMM_H(imm))
542 EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
543 break;
544 case BPF_ALU64 | BPF_XOR | BPF_X:
545 if (dst_reg == src_reg) {
546 EMIT(PPC_RAW_LI(dst_reg, 0));
547 EMIT(PPC_RAW_LI(dst_reg_h, 0));
548 } else {
549 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
550 EMIT(PPC_RAW_XOR(dst_reg_h, dst_reg_h, src_reg_h));
551 }
552 break;
553 case BPF_ALU | BPF_XOR | BPF_X:
554 if (dst_reg == src_reg)
555 EMIT(PPC_RAW_LI(dst_reg, 0));
556 else
557 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
558 break;
559 case BPF_ALU64 | BPF_XOR | BPF_K:
560 if (imm < 0)
561 EMIT(PPC_RAW_NOR(dst_reg_h, dst_reg_h, dst_reg_h));
562 fallthrough;
563 case BPF_ALU | BPF_XOR | BPF_K:
564 if (IMM_L(imm))
565 EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
566 if (IMM_H(imm))
567 EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
568 break;
569 case BPF_ALU | BPF_LSH | BPF_X:
570 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
571 break;
572 case BPF_ALU64 | BPF_LSH | BPF_X:
573 bpf_set_seen_register(ctx, tmp_reg);
574 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
575 EMIT(PPC_RAW_SLW(dst_reg_h, dst_reg_h, src_reg));
576 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
577 EMIT(PPC_RAW_SRW(_R0, dst_reg, _R0));
578 EMIT(PPC_RAW_SLW(tmp_reg, dst_reg, tmp_reg));
579 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, _R0));
580 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
581 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg));
582 break;
583 case BPF_ALU | BPF_LSH | BPF_K:
584 if (!imm)
585 break;
586 EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
587 break;
588 case BPF_ALU64 | BPF_LSH | BPF_K:
589 if (imm < 0)
590 return -EINVAL;
591 if (!imm)
592 break;
593 if (imm < 32) {
594 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, imm, 0, 31 - imm));
595 EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31));
596 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, imm, 0, 31 - imm));
597 break;
598 }
599 if (imm < 64)
600 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg, imm, 0, 31 - imm));
601 else
602 EMIT(PPC_RAW_LI(dst_reg_h, 0));
603 EMIT(PPC_RAW_LI(dst_reg, 0));
604 break;
605 case BPF_ALU | BPF_RSH | BPF_X:
606 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
607 break;
608 case BPF_ALU64 | BPF_RSH | BPF_X:
609 bpf_set_seen_register(ctx, tmp_reg);
610 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
611 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
612 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
613 EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0));
614 EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg));
615 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
616 EMIT(PPC_RAW_SRW(dst_reg_h, dst_reg_h, src_reg));
617 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
618 break;
619 case BPF_ALU | BPF_RSH | BPF_K:
620 if (!imm)
621 break;
622 EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
623 break;
624 case BPF_ALU64 | BPF_RSH | BPF_K:
625 if (imm < 0)
626 return -EINVAL;
627 if (!imm)
628 break;
629 if (imm < 32) {
630 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
631 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
632 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, 32 - imm, imm, 31));
633 break;
634 }
635 if (imm < 64)
636 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg_h, 64 - imm, imm - 32, 31));
637 else
638 EMIT(PPC_RAW_LI(dst_reg, 0));
639 EMIT(PPC_RAW_LI(dst_reg_h, 0));
640 break;
641 case BPF_ALU | BPF_ARSH | BPF_X:
642 EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
643 break;
644 case BPF_ALU64 | BPF_ARSH | BPF_X:
645 bpf_set_seen_register(ctx, tmp_reg);
646 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
647 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
648 EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0));
649 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
650 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
651 EMIT(PPC_RAW_RLWINM(_R0, tmp_reg, 0, 26, 26));
652 EMIT(PPC_RAW_SRAW(tmp_reg, dst_reg_h, tmp_reg));
653 EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg_h, src_reg));
654 EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, _R0));
655 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
656 break;
657 case BPF_ALU | BPF_ARSH | BPF_K:
658 if (!imm)
659 break;
660 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
661 break;
662 case BPF_ALU64 | BPF_ARSH | BPF_K:
663 if (imm < 0)
664 return -EINVAL;
665 if (!imm)
666 break;
667 if (imm < 32) {
668 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
669 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
670 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
671 break;
672 }
673 if (imm < 64)
674 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, imm - 32));
675 else
676 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, 31));
677 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, 31));
678 break;
679
680
681
682
683 case BPF_ALU64 | BPF_MOV | BPF_X:
684 if (dst_reg == src_reg)
685 break;
686 EMIT(PPC_RAW_MR(dst_reg, src_reg));
687 EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h));
688 break;
689 case BPF_ALU | BPF_MOV | BPF_X:
690
691 if (imm == 1)
692 EMIT(PPC_RAW_LI(dst_reg_h, 0));
693 else if (dst_reg != src_reg)
694 EMIT(PPC_RAW_MR(dst_reg, src_reg));
695 break;
696 case BPF_ALU64 | BPF_MOV | BPF_K:
697 PPC_LI32(dst_reg, imm);
698 PPC_EX32(dst_reg_h, imm);
699 break;
700 case BPF_ALU | BPF_MOV | BPF_K:
701 PPC_LI32(dst_reg, imm);
702 break;
703
704
705
706
707 case BPF_ALU | BPF_END | BPF_FROM_LE:
708 switch (imm) {
709 case 16:
710
711 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg, 16, 0, 15));
712
713 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31));
714 break;
715 case 32:
716
717
718
719
720
721 EMIT(PPC_RAW_RLWINM(_R0, dst_reg, 8, 0, 31));
722
723 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 0, 7));
724
725 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 16, 23));
726 EMIT(PPC_RAW_MR(dst_reg, _R0));
727 break;
728 case 64:
729 bpf_set_seen_register(ctx, tmp_reg);
730 EMIT(PPC_RAW_RLWINM(tmp_reg, dst_reg, 8, 0, 31));
731 EMIT(PPC_RAW_RLWINM(_R0, dst_reg_h, 8, 0, 31));
732
733 EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 0, 7));
734 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 0, 7));
735
736 EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 16, 23));
737 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 16, 23));
738 EMIT(PPC_RAW_MR(dst_reg, _R0));
739 EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg));
740 break;
741 }
742 break;
743 case BPF_ALU | BPF_END | BPF_FROM_BE:
744 switch (imm) {
745 case 16:
746
747 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 16, 31));
748 break;
749 case 32:
750 case 64:
751
752 break;
753 }
754 break;
755
756
757
758
759 case BPF_ST | BPF_NOSPEC:
760 break;
761
762
763
764
765 case BPF_STX | BPF_MEM | BPF_B:
766 EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
767 break;
768 case BPF_ST | BPF_MEM | BPF_B:
769 PPC_LI32(_R0, imm);
770 EMIT(PPC_RAW_STB(_R0, dst_reg, off));
771 break;
772 case BPF_STX | BPF_MEM | BPF_H:
773 EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
774 break;
775 case BPF_ST | BPF_MEM | BPF_H:
776 PPC_LI32(_R0, imm);
777 EMIT(PPC_RAW_STH(_R0, dst_reg, off));
778 break;
779 case BPF_STX | BPF_MEM | BPF_W:
780 EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
781 break;
782 case BPF_ST | BPF_MEM | BPF_W:
783 PPC_LI32(_R0, imm);
784 EMIT(PPC_RAW_STW(_R0, dst_reg, off));
785 break;
786 case BPF_STX | BPF_MEM | BPF_DW:
787 EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off));
788 EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4));
789 break;
790 case BPF_ST | BPF_MEM | BPF_DW:
791 PPC_LI32(_R0, imm);
792 EMIT(PPC_RAW_STW(_R0, dst_reg, off + 4));
793 PPC_EX32(_R0, imm);
794 EMIT(PPC_RAW_STW(_R0, dst_reg, off));
795 break;
796
797
798
799
800 case BPF_STX | BPF_ATOMIC | BPF_W:
801 if (imm != BPF_ADD) {
802 pr_err_ratelimited("eBPF filter atomic op code %02x (@%d) unsupported\n",
803 code, i);
804 return -ENOTSUPP;
805 }
806
807
808
809 bpf_set_seen_register(ctx, tmp_reg);
810
811 EMIT(PPC_RAW_LI(tmp_reg, off));
812
813 EMIT(PPC_RAW_LWARX(_R0, tmp_reg, dst_reg, 0));
814
815 EMIT(PPC_RAW_ADD(_R0, _R0, src_reg));
816
817 EMIT(PPC_RAW_STWCX(_R0, tmp_reg, dst_reg));
818
819 PPC_BCC_SHORT(COND_NE, (ctx->idx - 3) * 4);
820 break;
821
822 case BPF_STX | BPF_ATOMIC | BPF_DW:
823 return -EOPNOTSUPP;
824
825
826
827
828 case BPF_LDX | BPF_MEM | BPF_B:
829 case BPF_LDX | BPF_PROBE_MEM | BPF_B:
830 case BPF_LDX | BPF_MEM | BPF_H:
831 case BPF_LDX | BPF_PROBE_MEM | BPF_H:
832 case BPF_LDX | BPF_MEM | BPF_W:
833 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
834 case BPF_LDX | BPF_MEM | BPF_DW:
835 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
836
837
838
839
840
841
842 if (BPF_MODE(code) == BPF_PROBE_MEM) {
843 PPC_LI32(_R0, TASK_SIZE - off);
844 EMIT(PPC_RAW_CMPLW(src_reg, _R0));
845 PPC_BCC_SHORT(COND_GT, (ctx->idx + 4) * 4);
846 EMIT(PPC_RAW_LI(dst_reg, 0));
847
848
849
850
851
852
853
854
855 if (size == BPF_DW && !fp->aux->verifier_zext)
856 EMIT(PPC_RAW_LI(dst_reg_h, 0));
857 else
858 EMIT(PPC_RAW_NOP());
859
860
861
862
863
864 if (size == BPF_DW)
865 PPC_JMP((ctx->idx + 3) * 4);
866 else
867 PPC_JMP((ctx->idx + 2) * 4);
868 }
869
870 switch (size) {
871 case BPF_B:
872 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
873 break;
874 case BPF_H:
875 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
876 break;
877 case BPF_W:
878 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
879 break;
880 case BPF_DW:
881 EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off));
882 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4));
883 break;
884 }
885
886 if (size != BPF_DW && !fp->aux->verifier_zext)
887 EMIT(PPC_RAW_LI(dst_reg_h, 0));
888
889 if (BPF_MODE(code) == BPF_PROBE_MEM) {
890 int insn_idx = ctx->idx - 1;
891 int jmp_off = 4;
892
893
894
895
896
897
898
899
900
901
902
903
904 if (size == BPF_DW || !fp->aux->verifier_zext) {
905 insn_idx -= 1;
906 jmp_off += 4;
907 }
908
909 ret = bpf_add_extable_entry(fp, image, pass, ctx, insn_idx,
910 jmp_off, dst_reg);
911 if (ret)
912 return ret;
913 }
914 break;
915
916
917
918
919
920 case BPF_LD | BPF_IMM | BPF_DW:
921 tmp_idx = ctx->idx;
922 PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm);
923 PPC_LI32(dst_reg, (u32)insn[i].imm);
924
925 for (j = ctx->idx - tmp_idx; j < 4; j++)
926 EMIT(PPC_RAW_NOP());
927
928 addrs[++i] = ctx->idx * 4;
929 break;
930
931
932
933
934 case BPF_JMP | BPF_EXIT:
935
936
937
938
939
940 if (i != flen - 1) {
941 ret = bpf_jit_emit_exit_insn(image, ctx, _R0, exit_addr);
942 if (ret)
943 return ret;
944 }
945
946 break;
947
948
949
950
951 case BPF_JMP | BPF_CALL:
952 ctx->seen |= SEEN_FUNC;
953
954 ret = bpf_jit_get_func_addr(fp, &insn[i], false,
955 &func_addr, &func_addr_fixed);
956 if (ret < 0)
957 return ret;
958
959 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) {
960 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5) - 1, _R1, 8));
961 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5), _R1, 12));
962 }
963
964 ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr);
965 if (ret)
966 return ret;
967
968 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0) - 1, _R3));
969 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0), _R4));
970 break;
971
972
973
974
975 case BPF_JMP | BPF_JA:
976 PPC_JMP(addrs[i + 1 + off]);
977 break;
978
979 case BPF_JMP | BPF_JGT | BPF_K:
980 case BPF_JMP | BPF_JGT | BPF_X:
981 case BPF_JMP | BPF_JSGT | BPF_K:
982 case BPF_JMP | BPF_JSGT | BPF_X:
983 case BPF_JMP32 | BPF_JGT | BPF_K:
984 case BPF_JMP32 | BPF_JGT | BPF_X:
985 case BPF_JMP32 | BPF_JSGT | BPF_K:
986 case BPF_JMP32 | BPF_JSGT | BPF_X:
987 true_cond = COND_GT;
988 goto cond_branch;
989 case BPF_JMP | BPF_JLT | BPF_K:
990 case BPF_JMP | BPF_JLT | BPF_X:
991 case BPF_JMP | BPF_JSLT | BPF_K:
992 case BPF_JMP | BPF_JSLT | BPF_X:
993 case BPF_JMP32 | BPF_JLT | BPF_K:
994 case BPF_JMP32 | BPF_JLT | BPF_X:
995 case BPF_JMP32 | BPF_JSLT | BPF_K:
996 case BPF_JMP32 | BPF_JSLT | BPF_X:
997 true_cond = COND_LT;
998 goto cond_branch;
999 case BPF_JMP | BPF_JGE | BPF_K:
1000 case BPF_JMP | BPF_JGE | BPF_X:
1001 case BPF_JMP | BPF_JSGE | BPF_K:
1002 case BPF_JMP | BPF_JSGE | BPF_X:
1003 case BPF_JMP32 | BPF_JGE | BPF_K:
1004 case BPF_JMP32 | BPF_JGE | BPF_X:
1005 case BPF_JMP32 | BPF_JSGE | BPF_K:
1006 case BPF_JMP32 | BPF_JSGE | BPF_X:
1007 true_cond = COND_GE;
1008 goto cond_branch;
1009 case BPF_JMP | BPF_JLE | BPF_K:
1010 case BPF_JMP | BPF_JLE | BPF_X:
1011 case BPF_JMP | BPF_JSLE | BPF_K:
1012 case BPF_JMP | BPF_JSLE | BPF_X:
1013 case BPF_JMP32 | BPF_JLE | BPF_K:
1014 case BPF_JMP32 | BPF_JLE | BPF_X:
1015 case BPF_JMP32 | BPF_JSLE | BPF_K:
1016 case BPF_JMP32 | BPF_JSLE | BPF_X:
1017 true_cond = COND_LE;
1018 goto cond_branch;
1019 case BPF_JMP | BPF_JEQ | BPF_K:
1020 case BPF_JMP | BPF_JEQ | BPF_X:
1021 case BPF_JMP32 | BPF_JEQ | BPF_K:
1022 case BPF_JMP32 | BPF_JEQ | BPF_X:
1023 true_cond = COND_EQ;
1024 goto cond_branch;
1025 case BPF_JMP | BPF_JNE | BPF_K:
1026 case BPF_JMP | BPF_JNE | BPF_X:
1027 case BPF_JMP32 | BPF_JNE | BPF_K:
1028 case BPF_JMP32 | BPF_JNE | BPF_X:
1029 true_cond = COND_NE;
1030 goto cond_branch;
1031 case BPF_JMP | BPF_JSET | BPF_K:
1032 case BPF_JMP | BPF_JSET | BPF_X:
1033 case BPF_JMP32 | BPF_JSET | BPF_K:
1034 case BPF_JMP32 | BPF_JSET | BPF_X:
1035 true_cond = COND_NE;
1036
1037
1038cond_branch:
1039 switch (code) {
1040 case BPF_JMP | BPF_JGT | BPF_X:
1041 case BPF_JMP | BPF_JLT | BPF_X:
1042 case BPF_JMP | BPF_JGE | BPF_X:
1043 case BPF_JMP | BPF_JLE | BPF_X:
1044 case BPF_JMP | BPF_JEQ | BPF_X:
1045 case BPF_JMP | BPF_JNE | BPF_X:
1046
1047 EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h));
1048 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1049 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1050 break;
1051 case BPF_JMP32 | BPF_JGT | BPF_X:
1052 case BPF_JMP32 | BPF_JLT | BPF_X:
1053 case BPF_JMP32 | BPF_JGE | BPF_X:
1054 case BPF_JMP32 | BPF_JLE | BPF_X:
1055 case BPF_JMP32 | BPF_JEQ | BPF_X:
1056 case BPF_JMP32 | BPF_JNE | BPF_X:
1057
1058 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1059 break;
1060 case BPF_JMP | BPF_JSGT | BPF_X:
1061 case BPF_JMP | BPF_JSLT | BPF_X:
1062 case BPF_JMP | BPF_JSGE | BPF_X:
1063 case BPF_JMP | BPF_JSLE | BPF_X:
1064
1065 EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h));
1066 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1067 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1068 break;
1069 case BPF_JMP32 | BPF_JSGT | BPF_X:
1070 case BPF_JMP32 | BPF_JSLT | BPF_X:
1071 case BPF_JMP32 | BPF_JSGE | BPF_X:
1072 case BPF_JMP32 | BPF_JSLE | BPF_X:
1073
1074 EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
1075 break;
1076 case BPF_JMP | BPF_JSET | BPF_X:
1077 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg_h, src_reg_h));
1078 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1079 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1080 break;
1081 case BPF_JMP32 | BPF_JSET | BPF_X: {
1082 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1083 break;
1084 case BPF_JMP | BPF_JNE | BPF_K:
1085 case BPF_JMP | BPF_JEQ | BPF_K:
1086 case BPF_JMP | BPF_JGT | BPF_K:
1087 case BPF_JMP | BPF_JLT | BPF_K:
1088 case BPF_JMP | BPF_JGE | BPF_K:
1089 case BPF_JMP | BPF_JLE | BPF_K:
1090
1091
1092
1093
1094 if (imm >= 0 && imm < 32768) {
1095 EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0));
1096 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1097 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1098 } else {
1099
1100 PPC_EX32(_R0, imm);
1101 EMIT(PPC_RAW_CMPLW(dst_reg_h, _R0));
1102 PPC_LI32(_R0, imm);
1103 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1104 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1105 }
1106 break;
1107 case BPF_JMP32 | BPF_JNE | BPF_K:
1108 case BPF_JMP32 | BPF_JEQ | BPF_K:
1109 case BPF_JMP32 | BPF_JGT | BPF_K:
1110 case BPF_JMP32 | BPF_JLT | BPF_K:
1111 case BPF_JMP32 | BPF_JGE | BPF_K:
1112 case BPF_JMP32 | BPF_JLE | BPF_K:
1113 if (imm >= 0 && imm < 65536) {
1114 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1115 } else {
1116 PPC_LI32(_R0, imm);
1117 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1118 }
1119 break;
1120 }
1121 case BPF_JMP | BPF_JSGT | BPF_K:
1122 case BPF_JMP | BPF_JSLT | BPF_K:
1123 case BPF_JMP | BPF_JSGE | BPF_K:
1124 case BPF_JMP | BPF_JSLE | BPF_K:
1125 if (imm >= 0 && imm < 65536) {
1126 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1127 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1128 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1129 } else {
1130
1131 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1132 PPC_LI32(_R0, imm);
1133 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1134 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1135 }
1136 break;
1137 case BPF_JMP32 | BPF_JSGT | BPF_K:
1138 case BPF_JMP32 | BPF_JSLT | BPF_K:
1139 case BPF_JMP32 | BPF_JSGE | BPF_K:
1140 case BPF_JMP32 | BPF_JSLE | BPF_K:
1141
1142
1143
1144
1145 if (imm >= -32768 && imm < 32768) {
1146 EMIT(PPC_RAW_CMPWI(dst_reg, imm));
1147 } else {
1148
1149 PPC_LI32(_R0, imm);
1150 EMIT(PPC_RAW_CMPW(dst_reg, _R0));
1151 }
1152 break;
1153 case BPF_JMP | BPF_JSET | BPF_K:
1154
1155 if (imm >= 0 && imm < 32768) {
1156
1157 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1158 } else {
1159 PPC_LI32(_R0, imm);
1160 if (imm < 0) {
1161 EMIT(PPC_RAW_CMPWI(dst_reg_h, 0));
1162 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1163 }
1164 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1165 }
1166 break;
1167 case BPF_JMP32 | BPF_JSET | BPF_K:
1168
1169 if (imm >= 0 && imm < 32768) {
1170
1171 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1172 } else {
1173 PPC_LI32(_R0, imm);
1174 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1175 }
1176 break;
1177 }
1178 PPC_BCC(true_cond, addrs[i + 1 + off]);
1179 break;
1180
1181
1182
1183
1184 case BPF_JMP | BPF_TAIL_CALL:
1185 ctx->seen |= SEEN_TAILCALL;
1186 ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
1187 if (ret < 0)
1188 return ret;
1189 break;
1190
1191 default:
1192
1193
1194
1195
1196
1197 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i);
1198 return -EOPNOTSUPP;
1199 }
1200 if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext &&
1201 !insn_is_zext(&insn[i + 1]) && !(BPF_OP(code) == BPF_END && imm == 64))
1202 EMIT(PPC_RAW_LI(dst_reg_h, 0));
1203 }
1204
1205
1206 addrs[i] = ctx->idx * 4;
1207
1208 return 0;
1209}
1210