1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/math64.h>
16#include <linux/errno.h>
17#include <linux/filter.h>
18#include <linux/bpf.h>
19#include <asm/cpu-features.h>
20#include <asm/isa-rev.h>
21#include <asm/uasm.h>
22
23#include "bpf_jit_comp.h"
24
25
26#undef MIPS_R_A4
27#undef MIPS_R_A5
28#undef MIPS_R_A6
29#undef MIPS_R_A7
30
31
32#define MIPS_STACK_ALIGNMENT 8
33
34
35
36
37
38#define JIT_RESERVED_STACK 16
39
40
41#define JIT_REG_TMP MAX_BPF_JIT_REG
42
43
44
45
46
47
48#ifdef __BIG_ENDIAN
49#define JIT_TCALL_SKIP 12
50#else
51#define JIT_TCALL_SKIP 8
52#endif
53
54
55#define JIT_RETURN_REGS \
56 (BIT(MIPS_R_V0) | \
57 BIT(MIPS_R_V1))
58
59
60#define JIT_ARG_REGS \
61 (BIT(MIPS_R_A0) | \
62 BIT(MIPS_R_A1) | \
63 BIT(MIPS_R_A2) | \
64 BIT(MIPS_R_A3))
65
66
67#define JIT_STACK_REGS \
68 (BIT(MIPS_R_T0) | \
69 BIT(MIPS_R_T1) | \
70 BIT(MIPS_R_T2) | \
71 BIT(MIPS_R_T3) | \
72 BIT(MIPS_R_T4) | \
73 BIT(MIPS_R_T5))
74
75
76#define JIT_CALLER_REGS \
77 (JIT_RETURN_REGS | \
78 JIT_ARG_REGS | \
79 JIT_STACK_REGS)
80
81
82#define JIT_CALLEE_REGS \
83 (BIT(MIPS_R_S0) | \
84 BIT(MIPS_R_S1) | \
85 BIT(MIPS_R_S2) | \
86 BIT(MIPS_R_S3) | \
87 BIT(MIPS_R_S4) | \
88 BIT(MIPS_R_S5) | \
89 BIT(MIPS_R_S6) | \
90 BIT(MIPS_R_S7) | \
91 BIT(MIPS_R_GP) | \
92 BIT(MIPS_R_FP) | \
93 BIT(MIPS_R_RA))
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116static const u8 bpf2mips32[][2] = {
117
118 [BPF_REG_0] = {MIPS_R_V1, MIPS_R_V0},
119
120 [BPF_REG_1] = {MIPS_R_A1, MIPS_R_A0},
121 [BPF_REG_2] = {MIPS_R_A3, MIPS_R_A2},
122
123 [BPF_REG_3] = {MIPS_R_T1, MIPS_R_T0},
124 [BPF_REG_4] = {MIPS_R_T3, MIPS_R_T2},
125 [BPF_REG_5] = {MIPS_R_T5, MIPS_R_T4},
126
127 [BPF_REG_6] = {MIPS_R_S1, MIPS_R_S0},
128 [BPF_REG_7] = {MIPS_R_S3, MIPS_R_S2},
129 [BPF_REG_8] = {MIPS_R_S5, MIPS_R_S4},
130 [BPF_REG_9] = {MIPS_R_S7, MIPS_R_S6},
131
132#ifdef __BIG_ENDIAN
133 [BPF_REG_FP] = {MIPS_R_FP, MIPS_R_ZERO},
134#else
135 [BPF_REG_FP] = {MIPS_R_ZERO, MIPS_R_FP},
136#endif
137
138 [BPF_REG_AX] = {MIPS_R_GP, MIPS_R_AT},
139
140 [JIT_REG_TMP] = {MIPS_R_T7, MIPS_R_T6},
141};
142
143
144static inline u8 lo(const u8 reg[])
145{
146#ifdef __BIG_ENDIAN
147 return reg[0];
148#else
149 return reg[1];
150#endif
151}
152
153
154static inline u8 hi(const u8 reg[])
155{
156#ifdef __BIG_ENDIAN
157 return reg[1];
158#else
159 return reg[0];
160#endif
161}
162
163
164
165
166
167static void clobber_reg64(struct jit_context *ctx, const u8 reg[])
168{
169 clobber_reg(ctx, reg[0]);
170 clobber_reg(ctx, reg[1]);
171}
172
173
174static void emit_mov_se_i64(struct jit_context *ctx, const u8 dst[], s32 imm)
175{
176 emit_mov_i(ctx, lo(dst), imm);
177 if (imm < 0)
178 emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1);
179 else
180 emit(ctx, move, hi(dst), MIPS_R_ZERO);
181 clobber_reg64(ctx, dst);
182}
183
184
185static void emit_zext_ver(struct jit_context *ctx, const u8 dst[])
186{
187 if (!ctx->program->aux->verifier_zext) {
188 emit(ctx, move, hi(dst), MIPS_R_ZERO);
189 clobber_reg(ctx, hi(dst));
190 }
191}
192
193
194static void emit_load_delay(struct jit_context *ctx)
195{
196 if (!cpu_has_mips_2_3_4_5_r)
197 emit(ctx, nop);
198}
199
200
201static void emit_alu_i64(struct jit_context *ctx,
202 const u8 dst[], s32 imm, u8 op)
203{
204 u8 src = MIPS_R_T6;
205
206
207
208
209
210 if (imm > S32_MIN && imm < 0)
211 switch (op) {
212 case BPF_ADD:
213 op = BPF_SUB;
214 imm = -imm;
215 break;
216 case BPF_SUB:
217 op = BPF_ADD;
218 imm = -imm;
219 break;
220 }
221
222
223 emit_mov_i(ctx, src, imm);
224
225 switch (op) {
226
227 case BPF_ADD:
228 emit(ctx, addu, lo(dst), lo(dst), src);
229 emit(ctx, sltu, MIPS_R_T9, lo(dst), src);
230 emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9);
231 if (imm < 0)
232 emit(ctx, addiu, hi(dst), hi(dst), -1);
233 break;
234
235 case BPF_SUB:
236 emit(ctx, sltu, MIPS_R_T9, lo(dst), src);
237 emit(ctx, subu, lo(dst), lo(dst), src);
238 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9);
239 if (imm < 0)
240 emit(ctx, addiu, hi(dst), hi(dst), 1);
241 break;
242
243 case BPF_OR:
244 emit(ctx, or, lo(dst), lo(dst), src);
245 if (imm < 0)
246 emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1);
247 break;
248
249 case BPF_AND:
250 emit(ctx, and, lo(dst), lo(dst), src);
251 if (imm >= 0)
252 emit(ctx, move, hi(dst), MIPS_R_ZERO);
253 break;
254
255 case BPF_XOR:
256 emit(ctx, xor, lo(dst), lo(dst), src);
257 if (imm < 0) {
258 emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst));
259 emit(ctx, addiu, hi(dst), hi(dst), -1);
260 }
261 break;
262 }
263 clobber_reg64(ctx, dst);
264}
265
266
267static void emit_alu_r64(struct jit_context *ctx,
268 const u8 dst[], const u8 src[], u8 op)
269{
270 switch (BPF_OP(op)) {
271
272 case BPF_ADD:
273 if (src == dst) {
274 emit(ctx, srl, MIPS_R_T9, lo(dst), 31);
275 emit(ctx, addu, lo(dst), lo(dst), lo(dst));
276 } else {
277 emit(ctx, addu, lo(dst), lo(dst), lo(src));
278 emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src));
279 }
280 emit(ctx, addu, hi(dst), hi(dst), hi(src));
281 emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9);
282 break;
283
284 case BPF_SUB:
285 emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src));
286 emit(ctx, subu, lo(dst), lo(dst), lo(src));
287 emit(ctx, subu, hi(dst), hi(dst), hi(src));
288 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9);
289 break;
290
291 case BPF_OR:
292 emit(ctx, or, lo(dst), lo(dst), lo(src));
293 emit(ctx, or, hi(dst), hi(dst), hi(src));
294 break;
295
296 case BPF_AND:
297 emit(ctx, and, lo(dst), lo(dst), lo(src));
298 emit(ctx, and, hi(dst), hi(dst), hi(src));
299 break;
300
301 case BPF_XOR:
302 emit(ctx, xor, lo(dst), lo(dst), lo(src));
303 emit(ctx, xor, hi(dst), hi(dst), hi(src));
304 break;
305 }
306 clobber_reg64(ctx, dst);
307}
308
309
310static void emit_neg_i64(struct jit_context *ctx, const u8 dst[])
311{
312 emit(ctx, sltu, MIPS_R_T9, MIPS_R_ZERO, lo(dst));
313 emit(ctx, subu, lo(dst), MIPS_R_ZERO, lo(dst));
314 emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst));
315 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9);
316
317 clobber_reg64(ctx, dst);
318}
319
320
321static void emit_shift_i64(struct jit_context *ctx,
322 const u8 dst[], u32 imm, u8 op)
323{
324 switch (BPF_OP(op)) {
325
326 case BPF_LSH:
327 if (imm < 32) {
328 emit(ctx, srl, MIPS_R_T9, lo(dst), 32 - imm);
329 emit(ctx, sll, lo(dst), lo(dst), imm);
330 emit(ctx, sll, hi(dst), hi(dst), imm);
331 emit(ctx, or, hi(dst), hi(dst), MIPS_R_T9);
332 } else {
333 emit(ctx, sll, hi(dst), lo(dst), imm - 32);
334 emit(ctx, move, lo(dst), MIPS_R_ZERO);
335 }
336 break;
337
338 case BPF_RSH:
339 if (imm < 32) {
340 emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm);
341 emit(ctx, srl, lo(dst), lo(dst), imm);
342 emit(ctx, srl, hi(dst), hi(dst), imm);
343 emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9);
344 } else {
345 emit(ctx, srl, lo(dst), hi(dst), imm - 32);
346 emit(ctx, move, hi(dst), MIPS_R_ZERO);
347 }
348 break;
349
350 case BPF_ARSH:
351 if (imm < 32) {
352 emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm);
353 emit(ctx, srl, lo(dst), lo(dst), imm);
354 emit(ctx, sra, hi(dst), hi(dst), imm);
355 emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9);
356 } else {
357 emit(ctx, sra, lo(dst), hi(dst), imm - 32);
358 emit(ctx, sra, hi(dst), hi(dst), 31);
359 }
360 break;
361 }
362 clobber_reg64(ctx, dst);
363}
364
365
366static void emit_shift_r64(struct jit_context *ctx,
367 const u8 dst[], u8 src, u8 op)
368{
369 u8 t1 = MIPS_R_T8;
370 u8 t2 = MIPS_R_T9;
371
372 emit(ctx, andi, t1, src, 32);
373 emit(ctx, beqz, t1, 16);
374 emit(ctx, nor, t2, src, MIPS_R_ZERO);
375
376 switch (BPF_OP(op)) {
377
378 case BPF_LSH:
379
380 emit(ctx, sllv, hi(dst), lo(dst), src);
381 emit(ctx, move, lo(dst), MIPS_R_ZERO);
382 emit(ctx, b, 20);
383
384 emit(ctx, srl, t1, lo(dst), 1);
385 emit(ctx, srlv, t1, t1, t2);
386 emit(ctx, sllv, lo(dst), lo(dst), src);
387 emit(ctx, sllv, hi(dst), hi(dst), src);
388 emit(ctx, or, hi(dst), hi(dst), t1);
389 break;
390
391 case BPF_RSH:
392
393 emit(ctx, srlv, lo(dst), hi(dst), src);
394 emit(ctx, move, hi(dst), MIPS_R_ZERO);
395 emit(ctx, b, 20);
396
397 emit(ctx, sll, t1, hi(dst), 1);
398 emit(ctx, sllv, t1, t1, t2);
399 emit(ctx, srlv, lo(dst), lo(dst), src);
400 emit(ctx, srlv, hi(dst), hi(dst), src);
401 emit(ctx, or, lo(dst), lo(dst), t1);
402 break;
403
404 case BPF_ARSH:
405
406 emit(ctx, srav, lo(dst), hi(dst), src);
407 emit(ctx, sra, hi(dst), hi(dst), 31);
408 emit(ctx, b, 20);
409
410 emit(ctx, sll, t1, hi(dst), 1);
411 emit(ctx, sllv, t1, t1, t2);
412 emit(ctx, srlv, lo(dst), lo(dst), src);
413 emit(ctx, srav, hi(dst), hi(dst), src);
414 emit(ctx, or, lo(dst), lo(dst), t1);
415 break;
416 }
417
418
419 clobber_reg64(ctx, dst);
420}
421
422
423static void emit_mul_i64(struct jit_context *ctx, const u8 dst[], s32 imm)
424{
425 u8 src = MIPS_R_T6;
426 u8 tmp = MIPS_R_T9;
427
428 switch (imm) {
429
430 case 1:
431 break;
432
433 case -1:
434 emit_neg_i64(ctx, dst);
435 break;
436 case 0:
437 emit_mov_r(ctx, lo(dst), MIPS_R_ZERO);
438 emit_mov_r(ctx, hi(dst), MIPS_R_ZERO);
439 break;
440
441 default:
442
443 emit_mov_i(ctx, src, imm);
444 if (cpu_has_mips32r1 || cpu_has_mips32r6) {
445 emit(ctx, mul, hi(dst), hi(dst), src);
446 } else {
447 emit(ctx, multu, hi(dst), src);
448 emit(ctx, mflo, hi(dst));
449 }
450
451
452 if (imm < 0)
453 emit(ctx, subu, hi(dst), hi(dst), lo(dst));
454
455
456
457 if (cpu_has_mips32r6) {
458 emit(ctx, muhu, tmp, lo(dst), src);
459 emit(ctx, mulu, lo(dst), lo(dst), src);
460 } else {
461 emit(ctx, multu, lo(dst), src);
462 emit(ctx, mflo, lo(dst));
463 emit(ctx, mfhi, tmp);
464 }
465
466
467 emit(ctx, addu, hi(dst), hi(dst), tmp);
468 clobber_reg64(ctx, dst);
469 break;
470 }
471}
472
473
474static void emit_mul_r64(struct jit_context *ctx,
475 const u8 dst[], const u8 src[])
476{
477 u8 acc = MIPS_R_T8;
478 u8 tmp = MIPS_R_T9;
479
480
481 if (cpu_has_mips32r1 || cpu_has_mips32r6) {
482 emit(ctx, mul, acc, hi(dst), lo(src));
483 } else {
484 emit(ctx, multu, hi(dst), lo(src));
485 emit(ctx, mflo, acc);
486 }
487
488
489 if (cpu_has_mips32r1 || cpu_has_mips32r6) {
490 emit(ctx, mul, tmp, lo(dst), hi(src));
491 } else {
492 emit(ctx, multu, lo(dst), hi(src));
493 emit(ctx, mflo, tmp);
494 }
495
496
497 emit(ctx, addu, acc, acc, tmp);
498
499
500
501 if (cpu_has_mips32r6) {
502 emit(ctx, muhu, tmp, lo(dst), lo(src));
503 emit(ctx, mulu, lo(dst), lo(dst), lo(src));
504 } else {
505 emit(ctx, multu, lo(dst), lo(src));
506 emit(ctx, mflo, lo(dst));
507 emit(ctx, mfhi, tmp);
508 }
509
510
511 emit(ctx, addu, hi(dst), acc, tmp);
512 clobber_reg64(ctx, dst);
513}
514
515
516static u64 jit_mod64(u64 a, u64 b)
517{
518 u64 rem;
519
520 div64_u64_rem(a, b, &rem);
521 return rem;
522}
523
524
525static void emit_divmod_r64(struct jit_context *ctx,
526 const u8 dst[], const u8 src[], u8 op)
527{
528 const u8 *r0 = bpf2mips32[BPF_REG_0];
529 const u8 *r1 = bpf2mips32[BPF_REG_1];
530 const u8 *r2 = bpf2mips32[BPF_REG_2];
531 int exclude, k;
532 u32 addr = 0;
533
534
535 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
536 0, JIT_RESERVED_STACK);
537
538
539 for (k = 0; k < 2; k++) {
540 emit(ctx, move, MIPS_R_T9, src[k]);
541 emit(ctx, move, r1[k], dst[k]);
542 emit(ctx, move, r2[k], MIPS_R_T9);
543 }
544
545
546 switch (BPF_OP(op)) {
547
548 case BPF_DIV:
549 addr = (u32)&div64_u64;
550 break;
551
552 case BPF_MOD:
553 addr = (u32)&jit_mod64;
554 break;
555 }
556 emit_mov_i(ctx, MIPS_R_T9, addr);
557 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
558 emit(ctx, nop);
559
560
561 emit(ctx, move, dst[0], r0[0]);
562 emit(ctx, move, dst[1], r0[1]);
563
564
565 exclude = BIT(lo(dst)) | BIT(hi(dst));
566 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
567 exclude, JIT_RESERVED_STACK);
568 emit_load_delay(ctx);
569
570 clobber_reg64(ctx, dst);
571 clobber_reg(ctx, MIPS_R_V0);
572 clobber_reg(ctx, MIPS_R_V1);
573 clobber_reg(ctx, MIPS_R_RA);
574}
575
576
577static void emit_swap8_r(struct jit_context *ctx, u8 dst, u8 src, u8 mask)
578{
579 u8 tmp = MIPS_R_T9;
580
581 emit(ctx, and, tmp, src, mask);
582 emit(ctx, sll, tmp, tmp, 8);
583 emit(ctx, srl, dst, src, 8);
584 emit(ctx, and, dst, dst, mask);
585 emit(ctx, or, dst, dst, tmp);
586}
587
588
589static void emit_swap16_r(struct jit_context *ctx, u8 dst, u8 src)
590{
591 u8 tmp = MIPS_R_T9;
592
593 emit(ctx, sll, tmp, src, 16);
594 emit(ctx, srl, dst, src, 16);
595 emit(ctx, or, dst, dst, tmp);
596}
597
598
599static void emit_bswap_r64(struct jit_context *ctx, const u8 dst[], u32 width)
600{
601 u8 tmp = MIPS_R_T8;
602
603 switch (width) {
604
605 case 64:
606 if (cpu_has_mips32r2 || cpu_has_mips32r6) {
607 emit(ctx, rotr, tmp, hi(dst), 16);
608 emit(ctx, rotr, hi(dst), lo(dst), 16);
609 emit(ctx, wsbh, lo(dst), tmp);
610 emit(ctx, wsbh, hi(dst), hi(dst));
611 } else {
612 emit_swap16_r(ctx, tmp, lo(dst));
613 emit_swap16_r(ctx, lo(dst), hi(dst));
614 emit(ctx, move, hi(dst), tmp);
615
616 emit(ctx, lui, tmp, 0xff);
617 emit(ctx, ori, tmp, tmp, 0xff);
618 emit_swap8_r(ctx, lo(dst), lo(dst), tmp);
619 emit_swap8_r(ctx, hi(dst), hi(dst), tmp);
620 }
621 break;
622
623
624 case 32:
625 case 16:
626 emit_bswap_r(ctx, lo(dst), width);
627 emit(ctx, move, hi(dst), MIPS_R_ZERO);
628 break;
629 }
630 clobber_reg64(ctx, dst);
631}
632
633
634static void emit_trunc_r64(struct jit_context *ctx, const u8 dst[], u32 width)
635{
636 switch (width) {
637 case 64:
638 break;
639
640 case 32:
641 emit(ctx, move, hi(dst), MIPS_R_ZERO);
642 clobber_reg(ctx, hi(dst));
643 break;
644
645 case 16:
646 emit(ctx, move, hi(dst), MIPS_R_ZERO);
647 emit(ctx, andi, lo(dst), lo(dst), 0xffff);
648 clobber_reg64(ctx, dst);
649 break;
650 }
651}
652
653
654static void emit_ldx(struct jit_context *ctx,
655 const u8 dst[], u8 src, s16 off, u8 size)
656{
657 switch (size) {
658
659 case BPF_B:
660 emit(ctx, lbu, lo(dst), off, src);
661 emit(ctx, move, hi(dst), MIPS_R_ZERO);
662 break;
663
664 case BPF_H:
665 emit(ctx, lhu, lo(dst), off, src);
666 emit(ctx, move, hi(dst), MIPS_R_ZERO);
667 break;
668
669 case BPF_W:
670 emit(ctx, lw, lo(dst), off, src);
671 emit(ctx, move, hi(dst), MIPS_R_ZERO);
672 break;
673
674 case BPF_DW:
675 if (dst[1] == src) {
676 emit(ctx, lw, dst[0], off + 4, src);
677 emit(ctx, lw, dst[1], off, src);
678 } else {
679 emit(ctx, lw, dst[1], off, src);
680 emit(ctx, lw, dst[0], off + 4, src);
681 }
682 emit_load_delay(ctx);
683 break;
684 }
685 clobber_reg64(ctx, dst);
686}
687
688
689static void emit_stx(struct jit_context *ctx,
690 const u8 dst, const u8 src[], s16 off, u8 size)
691{
692 switch (size) {
693
694 case BPF_B:
695 emit(ctx, sb, lo(src), off, dst);
696 break;
697
698 case BPF_H:
699 emit(ctx, sh, lo(src), off, dst);
700 break;
701
702 case BPF_W:
703 emit(ctx, sw, lo(src), off, dst);
704 break;
705
706 case BPF_DW:
707 emit(ctx, sw, src[1], off, dst);
708 emit(ctx, sw, src[0], off + 4, dst);
709 break;
710 }
711}
712
713
714static void emit_atomic_r32(struct jit_context *ctx,
715 u8 dst, u8 src, s16 off, u8 code)
716{
717 u32 exclude = 0;
718 u32 addr = 0;
719
720
721 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
722 0, JIT_RESERVED_STACK);
723
724
725
726
727 emit(ctx, move, MIPS_R_T9, dst);
728 if (code == BPF_XCHG) {
729 emit(ctx, move, MIPS_R_A1, src);
730 emit(ctx, addiu, MIPS_R_A0, MIPS_R_T9, off);
731 } else {
732 emit(ctx, move, MIPS_R_A0, src);
733 emit(ctx, addiu, MIPS_R_A1, MIPS_R_T9, off);
734 }
735
736
737 switch (code) {
738 case BPF_ADD:
739 addr = (u32)&atomic_add;
740 break;
741 case BPF_ADD | BPF_FETCH:
742 addr = (u32)&atomic_fetch_add;
743 break;
744 case BPF_SUB:
745 addr = (u32)&atomic_sub;
746 break;
747 case BPF_SUB | BPF_FETCH:
748 addr = (u32)&atomic_fetch_sub;
749 break;
750 case BPF_OR:
751 addr = (u32)&atomic_or;
752 break;
753 case BPF_OR | BPF_FETCH:
754 addr = (u32)&atomic_fetch_or;
755 break;
756 case BPF_AND:
757 addr = (u32)&atomic_and;
758 break;
759 case BPF_AND | BPF_FETCH:
760 addr = (u32)&atomic_fetch_and;
761 break;
762 case BPF_XOR:
763 addr = (u32)&atomic_xor;
764 break;
765 case BPF_XOR | BPF_FETCH:
766 addr = (u32)&atomic_fetch_xor;
767 break;
768 case BPF_XCHG:
769 addr = (u32)&atomic_xchg;
770 break;
771 }
772 emit_mov_i(ctx, MIPS_R_T9, addr);
773 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
774 emit(ctx, nop);
775
776
777 if (code & BPF_FETCH) {
778 emit(ctx, move, src, MIPS_R_V0);
779 exclude = BIT(src);
780 clobber_reg(ctx, src);
781 }
782
783
784 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
785 exclude, JIT_RESERVED_STACK);
786 emit_load_delay(ctx);
787 clobber_reg(ctx, MIPS_R_RA);
788}
789
790
791static s64 jit_xchg64(s64 a, atomic64_t *v)
792{
793 return atomic64_xchg(v, a);
794}
795
796
797static void emit_atomic_r64(struct jit_context *ctx,
798 u8 dst, const u8 src[], s16 off, u8 code)
799{
800 const u8 *r0 = bpf2mips32[BPF_REG_0];
801 const u8 *r1 = bpf2mips32[BPF_REG_1];
802 u32 exclude = 0;
803 u32 addr = 0;
804
805
806 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
807 0, JIT_RESERVED_STACK);
808
809
810
811
812 emit(ctx, move, MIPS_R_T9, dst);
813 emit(ctx, move, r1[0], src[0]);
814 emit(ctx, move, r1[1], src[1]);
815 emit(ctx, addiu, MIPS_R_A2, MIPS_R_T9, off);
816
817
818 switch (code) {
819 case BPF_ADD:
820 addr = (u32)&atomic64_add;
821 break;
822 case BPF_ADD | BPF_FETCH:
823 addr = (u32)&atomic64_fetch_add;
824 break;
825 case BPF_SUB:
826 addr = (u32)&atomic64_sub;
827 break;
828 case BPF_SUB | BPF_FETCH:
829 addr = (u32)&atomic64_fetch_sub;
830 break;
831 case BPF_OR:
832 addr = (u32)&atomic64_or;
833 break;
834 case BPF_OR | BPF_FETCH:
835 addr = (u32)&atomic64_fetch_or;
836 break;
837 case BPF_AND:
838 addr = (u32)&atomic64_and;
839 break;
840 case BPF_AND | BPF_FETCH:
841 addr = (u32)&atomic64_fetch_and;
842 break;
843 case BPF_XOR:
844 addr = (u32)&atomic64_xor;
845 break;
846 case BPF_XOR | BPF_FETCH:
847 addr = (u32)&atomic64_fetch_xor;
848 break;
849 case BPF_XCHG:
850 addr = (u32)&jit_xchg64;
851 break;
852 }
853 emit_mov_i(ctx, MIPS_R_T9, addr);
854 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
855 emit(ctx, nop);
856
857
858 if (code & BPF_FETCH) {
859 emit(ctx, move, lo(src), lo(r0));
860 emit(ctx, move, hi(src), hi(r0));
861 exclude = BIT(src[0]) | BIT(src[1]);
862 clobber_reg64(ctx, src);
863 }
864
865
866 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
867 exclude, JIT_RESERVED_STACK);
868 emit_load_delay(ctx);
869 clobber_reg(ctx, MIPS_R_RA);
870}
871
872
873static void emit_cmpxchg_r32(struct jit_context *ctx, u8 dst, u8 src, s16 off)
874{
875 const u8 *r0 = bpf2mips32[BPF_REG_0];
876
877
878 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
879 JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32));
880
881
882
883
884
885 emit(ctx, addiu, MIPS_R_T9, dst, off);
886 emit(ctx, move, MIPS_R_T8, src);
887 emit(ctx, move, MIPS_R_A1, lo(r0));
888 emit(ctx, move, MIPS_R_A0, MIPS_R_T9);
889 emit(ctx, move, MIPS_R_A2, MIPS_R_T8);
890
891
892 emit_mov_i(ctx, MIPS_R_T9, (u32)&atomic_cmpxchg);
893 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
894 emit(ctx, nop);
895
896#ifdef __BIG_ENDIAN
897 emit(ctx, move, lo(r0), MIPS_R_V0);
898#endif
899
900 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
901 JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32));
902 emit_load_delay(ctx);
903 clobber_reg(ctx, MIPS_R_V0);
904 clobber_reg(ctx, MIPS_R_V1);
905 clobber_reg(ctx, MIPS_R_RA);
906}
907
908
909static void emit_cmpxchg_r64(struct jit_context *ctx,
910 u8 dst, const u8 src[], s16 off)
911{
912 const u8 *r0 = bpf2mips32[BPF_REG_0];
913 const u8 *r2 = bpf2mips32[BPF_REG_2];
914
915
916 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
917 JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32));
918
919
920
921
922
923 push_regs(ctx, BIT(src[0]) | BIT(src[1]), 0, JIT_RESERVED_STACK);
924 emit(ctx, addiu, MIPS_R_T9, dst, off);
925 emit(ctx, move, r2[0], r0[0]);
926 emit(ctx, move, r2[1], r0[1]);
927 emit(ctx, move, MIPS_R_A0, MIPS_R_T9);
928
929
930 emit_mov_i(ctx, MIPS_R_T9, (u32)&atomic64_cmpxchg);
931 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
932 emit(ctx, nop);
933
934
935 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
936 JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32));
937 emit_load_delay(ctx);
938 clobber_reg(ctx, MIPS_R_V0);
939 clobber_reg(ctx, MIPS_R_V1);
940 clobber_reg(ctx, MIPS_R_RA);
941}
942
943
944
945
946
947static void emit_movz_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt)
948{
949 if (cpu_has_mips_2) {
950 emit(ctx, movz, rd, rs, rt);
951 } else if (cpu_has_mips32r6) {
952 if (rs != MIPS_R_ZERO)
953 emit(ctx, seleqz, rs, rs, rt);
954 emit(ctx, selnez, rd, rd, rt);
955 if (rs != MIPS_R_ZERO)
956 emit(ctx, or, rd, rd, rs);
957 } else {
958 emit(ctx, bnez, rt, 8);
959 emit(ctx, nop);
960 emit(ctx, or, rd, rs, MIPS_R_ZERO);
961 }
962 clobber_reg(ctx, rd);
963 clobber_reg(ctx, rs);
964}
965
966
967
968
969
970static void emit_movn_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt)
971{
972 if (cpu_has_mips_2) {
973 emit(ctx, movn, rd, rs, rt);
974 } else if (cpu_has_mips32r6) {
975 if (rs != MIPS_R_ZERO)
976 emit(ctx, selnez, rs, rs, rt);
977 emit(ctx, seleqz, rd, rd, rt);
978 if (rs != MIPS_R_ZERO)
979 emit(ctx, or, rd, rd, rs);
980 } else {
981 emit(ctx, beqz, rt, 8);
982 emit(ctx, nop);
983 emit(ctx, or, rd, rs, MIPS_R_ZERO);
984 }
985 clobber_reg(ctx, rd);
986 clobber_reg(ctx, rs);
987}
988
989
990static void emit_sltiu_r64(struct jit_context *ctx, u8 rd,
991 const u8 rs[], s64 imm)
992{
993 u8 tmp = MIPS_R_T9;
994
995 if (imm < 0) {
996 emit_mov_i(ctx, rd, imm);
997 emit(ctx, sltu, rd, lo(rs), rd);
998 emit(ctx, sltiu, tmp, hi(rs), -1);
999 emit(ctx, or, rd, rd, tmp);
1000 } else {
1001 if (imm > 0x7fff) {
1002 emit_mov_i(ctx, rd, (s32)imm);
1003 emit(ctx, sltu, rd, lo(rs), rd);
1004 } else {
1005 emit(ctx, sltiu, rd, lo(rs), imm);
1006 }
1007 emit_movn_r(ctx, rd, MIPS_R_ZERO, hi(rs));
1008 }
1009}
1010
1011
1012static void emit_sltu_r64(struct jit_context *ctx, u8 rd,
1013 const u8 rs[], const u8 rt[])
1014{
1015 u8 tmp = MIPS_R_T9;
1016
1017 emit(ctx, sltu, rd, lo(rs), lo(rt));
1018 emit(ctx, subu, tmp, hi(rs), hi(rt));
1019 emit_movn_r(ctx, rd, MIPS_R_ZERO, tmp);
1020 emit(ctx, sltu, tmp, hi(rs), hi(rt));
1021 emit(ctx, or, rd, rd, tmp);
1022}
1023
1024
1025static void emit_slti_r64(struct jit_context *ctx, u8 rd,
1026 const u8 rs[], s64 imm)
1027{
1028 u8 t1 = MIPS_R_T8;
1029 u8 t2 = MIPS_R_T9;
1030 u8 cmp;
1031
1032
1033
1034
1035
1036 emit_mov_i(ctx, rd, (s32)imm);
1037 emit(ctx, sltu, t1, lo(rs), rd);
1038 emit(ctx, sltu, t2, rd, lo(rs));
1039 emit(ctx, srl, rd, hi(rs), 31);
1040 if (imm < 0)
1041 emit_movz_r(ctx, t1, t2, rd);
1042 else
1043 emit_movn_r(ctx, t1, t2, rd);
1044
1045
1046
1047
1048
1049 if (imm < 0) {
1050 emit(ctx, addiu, rd, hi(rs), 1);
1051 cmp = rd;
1052 } else {
1053 cmp = hi(rs);
1054 }
1055 emit_movn_r(ctx, t1, MIPS_R_ZERO, cmp);
1056
1057
1058
1059
1060
1061
1062 emit(ctx, slti, rd, hi(rs), imm < 0 ? -1 : 0);
1063 emit(ctx, or, rd, rd, t1);
1064}
1065
1066
1067static void emit_slt_r64(struct jit_context *ctx, u8 rd,
1068 const u8 rs[], const u8 rt[])
1069{
1070 u8 t1 = MIPS_R_T7;
1071 u8 t2 = MIPS_R_T8;
1072 u8 t3 = MIPS_R_T9;
1073
1074
1075
1076
1077
1078
1079 emit(ctx, sltu, t1, lo(rs), lo(rt));
1080 emit(ctx, sltu, t2, lo(rt), lo(rs));
1081 emit(ctx, xor, t3, hi(rs), hi(rt));
1082 emit(ctx, srl, rd, t3, 31);
1083 emit_movn_r(ctx, t1, t2, rd);
1084 emit_movn_r(ctx, t1, MIPS_R_ZERO, t3);
1085
1086
1087 emit(ctx, slt, rd, hi(rs), hi(rt));
1088 emit(ctx, or, rd, rd, t1);
1089}
1090
1091
1092static void emit_jmp_i64(struct jit_context *ctx,
1093 const u8 dst[], s32 imm, s32 off, u8 op)
1094{
1095 u8 tmp = MIPS_R_T6;
1096
1097 switch (op) {
1098
1099 case JIT_JNOP:
1100 break;
1101
1102
1103 case BPF_JEQ:
1104 case BPF_JNE:
1105 if (imm >= -0x7fff && imm <= 0x8000) {
1106 emit(ctx, addiu, tmp, lo(dst), -imm);
1107 } else if ((u32)imm <= 0xffff) {
1108 emit(ctx, xori, tmp, lo(dst), imm);
1109 } else {
1110 emit_mov_i(ctx, tmp, imm);
1111 emit(ctx, xor, tmp, lo(dst), tmp);
1112 }
1113 if (imm < 0) {
1114 emit(ctx, addu, MIPS_R_T9, hi(dst), 1);
1115 emit(ctx, or, tmp, tmp, MIPS_R_T9);
1116 } else {
1117 emit(ctx, or, tmp, tmp, hi(dst));
1118 }
1119 if (op == BPF_JEQ)
1120 emit(ctx, beqz, tmp, off);
1121 else
1122 emit(ctx, bnez, tmp, off);
1123 break;
1124
1125
1126 case BPF_JSET:
1127 case JIT_JNSET:
1128 if ((u32)imm <= 0xffff) {
1129 emit(ctx, andi, tmp, lo(dst), imm);
1130 } else {
1131 emit_mov_i(ctx, tmp, imm);
1132 emit(ctx, and, tmp, lo(dst), tmp);
1133 }
1134 if (imm < 0)
1135 emit(ctx, or, tmp, tmp, hi(dst));
1136 if (op == BPF_JSET)
1137 emit(ctx, bnez, tmp, off);
1138 else
1139 emit(ctx, beqz, tmp, off);
1140 break;
1141
1142 case BPF_JGT:
1143 emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1);
1144 emit(ctx, beqz, tmp, off);
1145 break;
1146
1147 case BPF_JGE:
1148 emit_sltiu_r64(ctx, tmp, dst, imm);
1149 emit(ctx, beqz, tmp, off);
1150 break;
1151
1152 case BPF_JLT:
1153 emit_sltiu_r64(ctx, tmp, dst, imm);
1154 emit(ctx, bnez, tmp, off);
1155 break;
1156
1157 case BPF_JLE:
1158 emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1);
1159 emit(ctx, bnez, tmp, off);
1160 break;
1161
1162 case BPF_JSGT:
1163 emit_slti_r64(ctx, tmp, dst, (s64)imm + 1);
1164 emit(ctx, beqz, tmp, off);
1165 break;
1166
1167 case BPF_JSGE:
1168 emit_slti_r64(ctx, tmp, dst, imm);
1169 emit(ctx, beqz, tmp, off);
1170 break;
1171
1172 case BPF_JSLT:
1173 emit_slti_r64(ctx, tmp, dst, imm);
1174 emit(ctx, bnez, tmp, off);
1175 break;
1176
1177 case BPF_JSLE:
1178 emit_slti_r64(ctx, tmp, dst, (s64)imm + 1);
1179 emit(ctx, bnez, tmp, off);
1180 break;
1181 }
1182}
1183
1184
1185static void emit_jmp_r64(struct jit_context *ctx,
1186 const u8 dst[], const u8 src[], s32 off, u8 op)
1187{
1188 u8 t1 = MIPS_R_T6;
1189 u8 t2 = MIPS_R_T7;
1190
1191 switch (op) {
1192
1193 case JIT_JNOP:
1194 break;
1195
1196
1197 case BPF_JEQ:
1198 case BPF_JNE:
1199 emit(ctx, subu, t1, lo(dst), lo(src));
1200 emit(ctx, subu, t2, hi(dst), hi(src));
1201 emit(ctx, or, t1, t1, t2);
1202 if (op == BPF_JEQ)
1203 emit(ctx, beqz, t1, off);
1204 else
1205 emit(ctx, bnez, t1, off);
1206 break;
1207
1208
1209 case BPF_JSET:
1210 case JIT_JNSET:
1211 emit(ctx, and, t1, lo(dst), lo(src));
1212 emit(ctx, and, t2, hi(dst), hi(src));
1213 emit(ctx, or, t1, t1, t2);
1214 if (op == BPF_JSET)
1215 emit(ctx, bnez, t1, off);
1216 else
1217 emit(ctx, beqz, t1, off);
1218 break;
1219
1220 case BPF_JGT:
1221 emit_sltu_r64(ctx, t1, src, dst);
1222 emit(ctx, bnez, t1, off);
1223 break;
1224
1225 case BPF_JGE:
1226 emit_sltu_r64(ctx, t1, dst, src);
1227 emit(ctx, beqz, t1, off);
1228 break;
1229
1230 case BPF_JLT:
1231 emit_sltu_r64(ctx, t1, dst, src);
1232 emit(ctx, bnez, t1, off);
1233 break;
1234
1235 case BPF_JLE:
1236 emit_sltu_r64(ctx, t1, src, dst);
1237 emit(ctx, beqz, t1, off);
1238 break;
1239
1240 case BPF_JSGT:
1241 emit_slt_r64(ctx, t1, src, dst);
1242 emit(ctx, bnez, t1, off);
1243 break;
1244
1245 case BPF_JSGE:
1246 emit_slt_r64(ctx, t1, dst, src);
1247 emit(ctx, beqz, t1, off);
1248 break;
1249
1250 case BPF_JSLT:
1251 emit_slt_r64(ctx, t1, dst, src);
1252 emit(ctx, bnez, t1, off);
1253 break;
1254
1255 case BPF_JSLE:
1256 emit_slt_r64(ctx, t1, src, dst);
1257 emit(ctx, beqz, t1, off);
1258 break;
1259 }
1260}
1261
1262
1263static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
1264{
1265 bool fixed;
1266 u64 addr;
1267
1268
1269 if (bpf_jit_get_func_addr(ctx->program, insn, false,
1270 &addr, &fixed) < 0)
1271 return -1;
1272 if (!fixed)
1273 return -1;
1274
1275
1276 push_regs(ctx, JIT_STACK_REGS, 0, JIT_RESERVED_STACK);
1277
1278
1279 emit_mov_i(ctx, MIPS_R_T9, addr);
1280 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
1281 emit(ctx, nop);
1282
1283 clobber_reg(ctx, MIPS_R_RA);
1284 clobber_reg(ctx, MIPS_R_V0);
1285 clobber_reg(ctx, MIPS_R_V1);
1286 return 0;
1287}
1288
1289
1290static int emit_tail_call(struct jit_context *ctx)
1291{
1292 u8 ary = lo(bpf2mips32[BPF_REG_2]);
1293 u8 ind = lo(bpf2mips32[BPF_REG_3]);
1294 u8 t1 = MIPS_R_T8;
1295 u8 t2 = MIPS_R_T9;
1296 int off;
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307 off = offsetof(struct bpf_array, map.max_entries);
1308 if (off > 0x7fff)
1309 return -1;
1310 emit(ctx, lw, t1, off, ary);
1311 emit_load_delay(ctx);
1312 emit(ctx, sltu, t1, ind, t1);
1313 emit(ctx, beqz, t1, get_offset(ctx, 1));
1314
1315
1316 emit(ctx, lw, t2, ctx->stack_size, MIPS_R_SP);
1317 emit_load_delay(ctx);
1318 emit(ctx, blez, t2, get_offset(ctx, 1));
1319 emit(ctx, addiu, t2, t2, -1);
1320 emit(ctx, sw, t2, ctx->stack_size, MIPS_R_SP);
1321
1322
1323 off = offsetof(struct bpf_array, ptrs);
1324 if (off > 0x7fff)
1325 return -1;
1326 emit(ctx, sll, t1, ind, 2);
1327 emit(ctx, addu, t1, t1, ary);
1328 emit(ctx, lw, t2, off, t1);
1329 emit_load_delay(ctx);
1330
1331
1332 emit(ctx, beqz, t2, get_offset(ctx, 1));
1333 emit(ctx, nop);
1334
1335
1336 off = offsetof(struct bpf_prog, bpf_func);
1337 if (off > 0x7fff)
1338 return -1;
1339 emit(ctx, lw, t1, off, t2);
1340 emit_load_delay(ctx);
1341 emit(ctx, addiu, t1, t1, JIT_TCALL_SKIP);
1342
1343
1344 build_epilogue(ctx, t1);
1345 return 0;
1346}
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373void build_prologue(struct jit_context *ctx)
1374{
1375 const u8 *r1 = bpf2mips32[BPF_REG_1];
1376 const u8 *fp = bpf2mips32[BPF_REG_FP];
1377 int stack, saved, locals, reserved;
1378
1379
1380
1381
1382
1383
1384 emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT, 0xffff));
1385 emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP);
1386
1387
1388
1389
1390
1391
1392#ifdef __BIG_ENDIAN
1393 emit(ctx, move, lo(r1), MIPS_R_A0);
1394#endif
1395
1396
1397
1398
1399 emit(ctx, move, hi(r1), MIPS_R_ZERO);
1400
1401
1402 if (ctx->accessed & BIT(BPF_REG_FP))
1403 clobber_reg64(ctx, fp);
1404
1405
1406 saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u32);
1407 saved = ALIGN(saved, MIPS_STACK_ALIGNMENT);
1408
1409
1410 locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT);
1411
1412
1413
1414
1415
1416
1417
1418 reserved = ctx->stack_used;
1419
1420
1421 stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT);
1422 emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, -stack);
1423
1424
1425 push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved);
1426
1427
1428 if (ctx->accessed & BIT(BPF_REG_FP))
1429 emit(ctx, addiu, lo(fp), MIPS_R_SP, stack - saved);
1430
1431 ctx->saved_size = saved;
1432 ctx->stack_size = stack;
1433}
1434
1435
1436void build_epilogue(struct jit_context *ctx, int dest_reg)
1437{
1438
1439 pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0,
1440 ctx->stack_size - ctx->saved_size);
1441
1442
1443
1444
1445#ifdef __BIG_ENDIAN
1446 emit(ctx, move, MIPS_R_V0, MIPS_R_V1);
1447#endif
1448
1449
1450 emit(ctx, jr, dest_reg);
1451 emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size);
1452}
1453
1454
1455int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
1456{
1457 const u8 *dst = bpf2mips32[insn->dst_reg];
1458 const u8 *src = bpf2mips32[insn->src_reg];
1459 const u8 *res = bpf2mips32[BPF_REG_0];
1460 const u8 *tmp = bpf2mips32[JIT_REG_TMP];
1461 u8 code = insn->code;
1462 s16 off = insn->off;
1463 s32 imm = insn->imm;
1464 s32 val, rel;
1465 u8 alu, jmp;
1466
1467 switch (code) {
1468
1469
1470 case BPF_ALU | BPF_MOV | BPF_K:
1471 emit_mov_i(ctx, lo(dst), imm);
1472 emit_zext_ver(ctx, dst);
1473 break;
1474
1475 case BPF_ALU | BPF_MOV | BPF_X:
1476 if (imm == 1) {
1477
1478 emit_mov_i(ctx, hi(dst), 0);
1479 } else {
1480 emit_mov_r(ctx, lo(dst), lo(src));
1481 emit_zext_ver(ctx, dst);
1482 }
1483 break;
1484
1485 case BPF_ALU | BPF_NEG:
1486 emit_alu_i(ctx, lo(dst), 0, BPF_NEG);
1487 emit_zext_ver(ctx, dst);
1488 break;
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500 case BPF_ALU | BPF_OR | BPF_K:
1501 case BPF_ALU | BPF_AND | BPF_K:
1502 case BPF_ALU | BPF_XOR | BPF_K:
1503 case BPF_ALU | BPF_LSH | BPF_K:
1504 case BPF_ALU | BPF_RSH | BPF_K:
1505 case BPF_ALU | BPF_ARSH | BPF_K:
1506 case BPF_ALU | BPF_ADD | BPF_K:
1507 case BPF_ALU | BPF_SUB | BPF_K:
1508 case BPF_ALU | BPF_MUL | BPF_K:
1509 case BPF_ALU | BPF_DIV | BPF_K:
1510 case BPF_ALU | BPF_MOD | BPF_K:
1511 if (!valid_alu_i(BPF_OP(code), imm)) {
1512 emit_mov_i(ctx, MIPS_R_T6, imm);
1513 emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code));
1514 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
1515 emit_alu_i(ctx, lo(dst), val, alu);
1516 }
1517 emit_zext_ver(ctx, dst);
1518 break;
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530 case BPF_ALU | BPF_AND | BPF_X:
1531 case BPF_ALU | BPF_OR | BPF_X:
1532 case BPF_ALU | BPF_XOR | BPF_X:
1533 case BPF_ALU | BPF_LSH | BPF_X:
1534 case BPF_ALU | BPF_RSH | BPF_X:
1535 case BPF_ALU | BPF_ARSH | BPF_X:
1536 case BPF_ALU | BPF_ADD | BPF_X:
1537 case BPF_ALU | BPF_SUB | BPF_X:
1538 case BPF_ALU | BPF_MUL | BPF_X:
1539 case BPF_ALU | BPF_DIV | BPF_X:
1540 case BPF_ALU | BPF_MOD | BPF_X:
1541 emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code));
1542 emit_zext_ver(ctx, dst);
1543 break;
1544
1545 case BPF_ALU64 | BPF_MOV | BPF_K:
1546 emit_mov_se_i64(ctx, dst, imm);
1547 break;
1548
1549 case BPF_ALU64 | BPF_MOV | BPF_X:
1550 emit_mov_r(ctx, lo(dst), lo(src));
1551 emit_mov_r(ctx, hi(dst), hi(src));
1552 break;
1553
1554 case BPF_ALU64 | BPF_NEG:
1555 emit_neg_i64(ctx, dst);
1556 break;
1557
1558 case BPF_ALU64 | BPF_AND | BPF_K:
1559 emit_alu_i64(ctx, dst, imm, BPF_OP(code));
1560 break;
1561
1562
1563
1564
1565 case BPF_ALU64 | BPF_OR | BPF_K:
1566 case BPF_ALU64 | BPF_XOR | BPF_K:
1567 case BPF_ALU64 | BPF_ADD | BPF_K:
1568 case BPF_ALU64 | BPF_SUB | BPF_K:
1569 if (imm)
1570 emit_alu_i64(ctx, dst, imm, BPF_OP(code));
1571 break;
1572
1573
1574
1575 case BPF_ALU64 | BPF_LSH | BPF_K:
1576 case BPF_ALU64 | BPF_RSH | BPF_K:
1577 case BPF_ALU64 | BPF_ARSH | BPF_K:
1578 if (imm)
1579 emit_shift_i64(ctx, dst, imm, BPF_OP(code));
1580 break;
1581
1582 case BPF_ALU64 | BPF_MUL | BPF_K:
1583 emit_mul_i64(ctx, dst, imm);
1584 break;
1585
1586
1587 case BPF_ALU64 | BPF_DIV | BPF_K:
1588 case BPF_ALU64 | BPF_MOD | BPF_K:
1589
1590
1591
1592
1593 emit_mov_se_i64(ctx, tmp, imm);
1594 emit_divmod_r64(ctx, dst, tmp, BPF_OP(code));
1595 break;
1596
1597
1598
1599
1600
1601 case BPF_ALU64 | BPF_AND | BPF_X:
1602 case BPF_ALU64 | BPF_OR | BPF_X:
1603 case BPF_ALU64 | BPF_XOR | BPF_X:
1604 case BPF_ALU64 | BPF_ADD | BPF_X:
1605 case BPF_ALU64 | BPF_SUB | BPF_X:
1606 emit_alu_r64(ctx, dst, src, BPF_OP(code));
1607 break;
1608
1609
1610
1611 case BPF_ALU64 | BPF_LSH | BPF_X:
1612 case BPF_ALU64 | BPF_RSH | BPF_X:
1613 case BPF_ALU64 | BPF_ARSH | BPF_X:
1614 emit_shift_r64(ctx, dst, lo(src), BPF_OP(code));
1615 break;
1616
1617 case BPF_ALU64 | BPF_MUL | BPF_X:
1618 emit_mul_r64(ctx, dst, src);
1619 break;
1620
1621
1622 case BPF_ALU64 | BPF_DIV | BPF_X:
1623 case BPF_ALU64 | BPF_MOD | BPF_X:
1624 emit_divmod_r64(ctx, dst, src, BPF_OP(code));
1625 break;
1626
1627
1628 case BPF_ALU | BPF_END | BPF_FROM_LE:
1629 case BPF_ALU | BPF_END | BPF_FROM_BE:
1630 if (BPF_SRC(code) ==
1631#ifdef __BIG_ENDIAN
1632 BPF_FROM_LE
1633#else
1634 BPF_FROM_BE
1635#endif
1636 )
1637 emit_bswap_r64(ctx, dst, imm);
1638 else
1639 emit_trunc_r64(ctx, dst, imm);
1640 break;
1641
1642 case BPF_LD | BPF_IMM | BPF_DW:
1643 emit_mov_i(ctx, lo(dst), imm);
1644 emit_mov_i(ctx, hi(dst), insn[1].imm);
1645 return 1;
1646
1647 case BPF_LDX | BPF_MEM | BPF_W:
1648 case BPF_LDX | BPF_MEM | BPF_H:
1649 case BPF_LDX | BPF_MEM | BPF_B:
1650 case BPF_LDX | BPF_MEM | BPF_DW:
1651 emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code));
1652 break;
1653
1654 case BPF_ST | BPF_MEM | BPF_W:
1655 case BPF_ST | BPF_MEM | BPF_H:
1656 case BPF_ST | BPF_MEM | BPF_B:
1657 case BPF_ST | BPF_MEM | BPF_DW:
1658 switch (BPF_SIZE(code)) {
1659 case BPF_DW:
1660
1661 emit_mov_se_i64(ctx, tmp, imm);
1662 break;
1663 case BPF_W:
1664 case BPF_H:
1665 case BPF_B:
1666 emit_mov_i(ctx, lo(tmp), imm);
1667 break;
1668 }
1669 emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code));
1670 break;
1671
1672 case BPF_STX | BPF_MEM | BPF_W:
1673 case BPF_STX | BPF_MEM | BPF_H:
1674 case BPF_STX | BPF_MEM | BPF_B:
1675 case BPF_STX | BPF_MEM | BPF_DW:
1676 emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code));
1677 break;
1678
1679 case BPF_ST | BPF_NOSPEC:
1680 break;
1681
1682 case BPF_STX | BPF_ATOMIC | BPF_W:
1683 switch (imm) {
1684 case BPF_ADD:
1685 case BPF_ADD | BPF_FETCH:
1686 case BPF_AND:
1687 case BPF_AND | BPF_FETCH:
1688 case BPF_OR:
1689 case BPF_OR | BPF_FETCH:
1690 case BPF_XOR:
1691 case BPF_XOR | BPF_FETCH:
1692 case BPF_XCHG:
1693 if (cpu_has_llsc)
1694 emit_atomic_r(ctx, lo(dst), lo(src), off, imm);
1695 else
1696 emit_atomic_r32(ctx, lo(dst), lo(src),
1697 off, imm);
1698 if (imm & BPF_FETCH)
1699 emit_zext_ver(ctx, src);
1700 break;
1701 case BPF_CMPXCHG:
1702 if (cpu_has_llsc)
1703 emit_cmpxchg_r(ctx, lo(dst), lo(src),
1704 lo(res), off);
1705 else
1706 emit_cmpxchg_r32(ctx, lo(dst), lo(src), off);
1707
1708 break;
1709 default:
1710 goto notyet;
1711 }
1712 break;
1713
1714 case BPF_STX | BPF_ATOMIC | BPF_DW:
1715 switch (imm) {
1716 case BPF_ADD:
1717 case BPF_ADD | BPF_FETCH:
1718 case BPF_AND:
1719 case BPF_AND | BPF_FETCH:
1720 case BPF_OR:
1721 case BPF_OR | BPF_FETCH:
1722 case BPF_XOR:
1723 case BPF_XOR | BPF_FETCH:
1724 case BPF_XCHG:
1725 emit_atomic_r64(ctx, lo(dst), src, off, imm);
1726 break;
1727 case BPF_CMPXCHG:
1728 emit_cmpxchg_r64(ctx, lo(dst), src, off);
1729 break;
1730 default:
1731 goto notyet;
1732 }
1733 break;
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745 case BPF_JMP32 | BPF_JEQ | BPF_X:
1746 case BPF_JMP32 | BPF_JNE | BPF_X:
1747 case BPF_JMP32 | BPF_JSET | BPF_X:
1748 case BPF_JMP32 | BPF_JGT | BPF_X:
1749 case BPF_JMP32 | BPF_JGE | BPF_X:
1750 case BPF_JMP32 | BPF_JLT | BPF_X:
1751 case BPF_JMP32 | BPF_JLE | BPF_X:
1752 case BPF_JMP32 | BPF_JSGT | BPF_X:
1753 case BPF_JMP32 | BPF_JSGE | BPF_X:
1754 case BPF_JMP32 | BPF_JSLT | BPF_X:
1755 case BPF_JMP32 | BPF_JSLE | BPF_X:
1756 if (off == 0)
1757 break;
1758 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
1759 emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp);
1760 if (finish_jmp(ctx, jmp, off) < 0)
1761 goto toofar;
1762 break;
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774 case BPF_JMP32 | BPF_JEQ | BPF_K:
1775 case BPF_JMP32 | BPF_JNE | BPF_K:
1776 case BPF_JMP32 | BPF_JSET | BPF_K:
1777 case BPF_JMP32 | BPF_JGT | BPF_K:
1778 case BPF_JMP32 | BPF_JGE | BPF_K:
1779 case BPF_JMP32 | BPF_JLT | BPF_K:
1780 case BPF_JMP32 | BPF_JLE | BPF_K:
1781 case BPF_JMP32 | BPF_JSGT | BPF_K:
1782 case BPF_JMP32 | BPF_JSGE | BPF_K:
1783 case BPF_JMP32 | BPF_JSLT | BPF_K:
1784 case BPF_JMP32 | BPF_JSLE | BPF_K:
1785 if (off == 0)
1786 break;
1787 setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel);
1788 if (valid_jmp_i(jmp, imm)) {
1789 emit_jmp_i(ctx, lo(dst), imm, rel, jmp);
1790 } else {
1791
1792 emit_mov_i(ctx, MIPS_R_T6, imm);
1793 emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp);
1794 }
1795 if (finish_jmp(ctx, jmp, off) < 0)
1796 goto toofar;
1797 break;
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809 case BPF_JMP | BPF_JEQ | BPF_X:
1810 case BPF_JMP | BPF_JNE | BPF_X:
1811 case BPF_JMP | BPF_JSET | BPF_X:
1812 case BPF_JMP | BPF_JGT | BPF_X:
1813 case BPF_JMP | BPF_JGE | BPF_X:
1814 case BPF_JMP | BPF_JLT | BPF_X:
1815 case BPF_JMP | BPF_JLE | BPF_X:
1816 case BPF_JMP | BPF_JSGT | BPF_X:
1817 case BPF_JMP | BPF_JSGE | BPF_X:
1818 case BPF_JMP | BPF_JSLT | BPF_X:
1819 case BPF_JMP | BPF_JSLE | BPF_X:
1820 if (off == 0)
1821 break;
1822 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
1823 emit_jmp_r64(ctx, dst, src, rel, jmp);
1824 if (finish_jmp(ctx, jmp, off) < 0)
1825 goto toofar;
1826 break;
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838 case BPF_JMP | BPF_JEQ | BPF_K:
1839 case BPF_JMP | BPF_JNE | BPF_K:
1840 case BPF_JMP | BPF_JSET | BPF_K:
1841 case BPF_JMP | BPF_JGT | BPF_K:
1842 case BPF_JMP | BPF_JGE | BPF_K:
1843 case BPF_JMP | BPF_JLT | BPF_K:
1844 case BPF_JMP | BPF_JLE | BPF_K:
1845 case BPF_JMP | BPF_JSGT | BPF_K:
1846 case BPF_JMP | BPF_JSGE | BPF_K:
1847 case BPF_JMP | BPF_JSLT | BPF_K:
1848 case BPF_JMP | BPF_JSLE | BPF_K:
1849 if (off == 0)
1850 break;
1851 setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel);
1852 emit_jmp_i64(ctx, dst, imm, rel, jmp);
1853 if (finish_jmp(ctx, jmp, off) < 0)
1854 goto toofar;
1855 break;
1856
1857 case BPF_JMP | BPF_JA:
1858 if (off == 0)
1859 break;
1860 if (emit_ja(ctx, off) < 0)
1861 goto toofar;
1862 break;
1863
1864 case BPF_JMP | BPF_TAIL_CALL:
1865 if (emit_tail_call(ctx) < 0)
1866 goto invalid;
1867 break;
1868
1869 case BPF_JMP | BPF_CALL:
1870 if (emit_call(ctx, insn) < 0)
1871 goto invalid;
1872 break;
1873
1874 case BPF_JMP | BPF_EXIT:
1875
1876
1877
1878
1879 if (ctx->bpf_index == ctx->program->len - 1)
1880 break;
1881 if (emit_exit(ctx) < 0)
1882 goto toofar;
1883 break;
1884
1885 default:
1886invalid:
1887 pr_err_once("unknown opcode %02x\n", code);
1888 return -EINVAL;
1889notyet:
1890 pr_info_once("*** NOT YET: opcode %02x ***\n", code);
1891 return -EFAULT;
1892toofar:
1893 pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n",
1894 ctx->bpf_index, code);
1895 return -E2BIG;
1896 }
1897 return 0;
1898}
1899