linux/arch/sparc/net/bpf_jit_comp_64.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2#include <linux/moduleloader.h>
   3#include <linux/workqueue.h>
   4#include <linux/netdevice.h>
   5#include <linux/filter.h>
   6#include <linux/bpf.h>
   7#include <linux/cache.h>
   8#include <linux/if_vlan.h>
   9
  10#include <asm/cacheflush.h>
  11#include <asm/ptrace.h>
  12
  13#include "bpf_jit_64.h"
  14
  15static inline bool is_simm13(unsigned int value)
  16{
  17        return value + 0x1000 < 0x2000;
  18}
  19
  20static inline bool is_simm10(unsigned int value)
  21{
  22        return value + 0x200 < 0x400;
  23}
  24
  25static inline bool is_simm5(unsigned int value)
  26{
  27        return value + 0x10 < 0x20;
  28}
  29
  30static inline bool is_sethi(unsigned int value)
  31{
  32        return (value & ~0x3fffff) == 0;
  33}
  34
  35static void bpf_flush_icache(void *start_, void *end_)
  36{
  37        /* Cheetah's I-cache is fully coherent.  */
  38        if (tlb_type == spitfire) {
  39                unsigned long start = (unsigned long) start_;
  40                unsigned long end = (unsigned long) end_;
  41
  42                start &= ~7UL;
  43                end = (end + 7UL) & ~7UL;
  44                while (start < end) {
  45                        flushi(start);
  46                        start += 32;
  47                }
  48        }
  49}
  50
  51#define S13(X)          ((X) & 0x1fff)
  52#define S5(X)           ((X) & 0x1f)
  53#define IMMED           0x00002000
  54#define RD(X)           ((X) << 25)
  55#define RS1(X)          ((X) << 14)
  56#define RS2(X)          ((X))
  57#define OP(X)           ((X) << 30)
  58#define OP2(X)          ((X) << 22)
  59#define OP3(X)          ((X) << 19)
  60#define COND(X)         (((X) & 0xf) << 25)
  61#define CBCOND(X)       (((X) & 0x1f) << 25)
  62#define F1(X)           OP(X)
  63#define F2(X, Y)        (OP(X) | OP2(Y))
  64#define F3(X, Y)        (OP(X) | OP3(Y))
  65#define ASI(X)          (((X) & 0xff) << 5)
  66
  67#define CONDN           COND(0x0)
  68#define CONDE           COND(0x1)
  69#define CONDLE          COND(0x2)
  70#define CONDL           COND(0x3)
  71#define CONDLEU         COND(0x4)
  72#define CONDCS          COND(0x5)
  73#define CONDNEG         COND(0x6)
  74#define CONDVC          COND(0x7)
  75#define CONDA           COND(0x8)
  76#define CONDNE          COND(0x9)
  77#define CONDG           COND(0xa)
  78#define CONDGE          COND(0xb)
  79#define CONDGU          COND(0xc)
  80#define CONDCC          COND(0xd)
  81#define CONDPOS         COND(0xe)
  82#define CONDVS          COND(0xf)
  83
  84#define CONDGEU         CONDCC
  85#define CONDLU          CONDCS
  86
  87#define WDISP22(X)      (((X) >> 2) & 0x3fffff)
  88#define WDISP19(X)      (((X) >> 2) & 0x7ffff)
  89
  90/* The 10-bit branch displacement for CBCOND is split into two fields */
  91static u32 WDISP10(u32 off)
  92{
  93        u32 ret = ((off >> 2) & 0xff) << 5;
  94
  95        ret |= ((off >> (2 + 8)) & 0x03) << 19;
  96
  97        return ret;
  98}
  99
 100#define CBCONDE         CBCOND(0x09)
 101#define CBCONDLE        CBCOND(0x0a)
 102#define CBCONDL         CBCOND(0x0b)
 103#define CBCONDLEU       CBCOND(0x0c)
 104#define CBCONDCS        CBCOND(0x0d)
 105#define CBCONDN         CBCOND(0x0e)
 106#define CBCONDVS        CBCOND(0x0f)
 107#define CBCONDNE        CBCOND(0x19)
 108#define CBCONDG         CBCOND(0x1a)
 109#define CBCONDGE        CBCOND(0x1b)
 110#define CBCONDGU        CBCOND(0x1c)
 111#define CBCONDCC        CBCOND(0x1d)
 112#define CBCONDPOS       CBCOND(0x1e)
 113#define CBCONDVC        CBCOND(0x1f)
 114
 115#define CBCONDGEU       CBCONDCC
 116#define CBCONDLU        CBCONDCS
 117
 118#define ANNUL           (1 << 29)
 119#define XCC             (1 << 21)
 120
 121#define BRANCH          (F2(0, 1) | XCC)
 122#define CBCOND_OP       (F2(0, 3) | XCC)
 123
 124#define BA              (BRANCH | CONDA)
 125#define BG              (BRANCH | CONDG)
 126#define BL              (BRANCH | CONDL)
 127#define BLE             (BRANCH | CONDLE)
 128#define BGU             (BRANCH | CONDGU)
 129#define BLEU            (BRANCH | CONDLEU)
 130#define BGE             (BRANCH | CONDGE)
 131#define BGEU            (BRANCH | CONDGEU)
 132#define BLU             (BRANCH | CONDLU)
 133#define BE              (BRANCH | CONDE)
 134#define BNE             (BRANCH | CONDNE)
 135
 136#define SETHI(K, REG)   \
 137        (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
 138#define OR_LO(K, REG)   \
 139        (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG))
 140
 141#define ADD             F3(2, 0x00)
 142#define AND             F3(2, 0x01)
 143#define ANDCC           F3(2, 0x11)
 144#define OR              F3(2, 0x02)
 145#define XOR             F3(2, 0x03)
 146#define SUB             F3(2, 0x04)
 147#define SUBCC           F3(2, 0x14)
 148#define MUL             F3(2, 0x0a)
 149#define MULX            F3(2, 0x09)
 150#define UDIVX           F3(2, 0x0d)
 151#define DIV             F3(2, 0x0e)
 152#define SLL             F3(2, 0x25)
 153#define SLLX            (F3(2, 0x25)|(1<<12))
 154#define SRA             F3(2, 0x27)
 155#define SRAX            (F3(2, 0x27)|(1<<12))
 156#define SRL             F3(2, 0x26)
 157#define SRLX            (F3(2, 0x26)|(1<<12))
 158#define JMPL            F3(2, 0x38)
 159#define SAVE            F3(2, 0x3c)
 160#define RESTORE         F3(2, 0x3d)
 161#define CALL            F1(1)
 162#define BR              F2(0, 0x01)
 163#define RD_Y            F3(2, 0x28)
 164#define WR_Y            F3(2, 0x30)
 165
 166#define LD32            F3(3, 0x00)
 167#define LD8             F3(3, 0x01)
 168#define LD16            F3(3, 0x02)
 169#define LD64            F3(3, 0x0b)
 170#define LD64A           F3(3, 0x1b)
 171#define ST8             F3(3, 0x05)
 172#define ST16            F3(3, 0x06)
 173#define ST32            F3(3, 0x04)
 174#define ST64            F3(3, 0x0e)
 175
 176#define CAS             F3(3, 0x3c)
 177#define CASX            F3(3, 0x3e)
 178
 179#define LDPTR           LD64
 180#define BASE_STACKFRAME 176
 181
 182#define LD32I           (LD32 | IMMED)
 183#define LD8I            (LD8 | IMMED)
 184#define LD16I           (LD16 | IMMED)
 185#define LD64I           (LD64 | IMMED)
 186#define LDPTRI          (LDPTR | IMMED)
 187#define ST32I           (ST32 | IMMED)
 188
 189struct jit_ctx {
 190        struct bpf_prog         *prog;
 191        unsigned int            *offset;
 192        int                     idx;
 193        int                     epilogue_offset;
 194        bool                    tmp_1_used;
 195        bool                    tmp_2_used;
 196        bool                    tmp_3_used;
 197        bool                    saw_frame_pointer;
 198        bool                    saw_call;
 199        bool                    saw_tail_call;
 200        u32                     *image;
 201};
 202
 203#define TMP_REG_1       (MAX_BPF_JIT_REG + 0)
 204#define TMP_REG_2       (MAX_BPF_JIT_REG + 1)
 205#define TMP_REG_3       (MAX_BPF_JIT_REG + 2)
 206
 207/* Map BPF registers to SPARC registers */
 208static const int bpf2sparc[] = {
 209        /* return value from in-kernel function, and exit value from eBPF */
 210        [BPF_REG_0] = O5,
 211
 212        /* arguments from eBPF program to in-kernel function */
 213        [BPF_REG_1] = O0,
 214        [BPF_REG_2] = O1,
 215        [BPF_REG_3] = O2,
 216        [BPF_REG_4] = O3,
 217        [BPF_REG_5] = O4,
 218
 219        /* callee saved registers that in-kernel function will preserve */
 220        [BPF_REG_6] = L0,
 221        [BPF_REG_7] = L1,
 222        [BPF_REG_8] = L2,
 223        [BPF_REG_9] = L3,
 224
 225        /* read-only frame pointer to access stack */
 226        [BPF_REG_FP] = L6,
 227
 228        [BPF_REG_AX] = G7,
 229
 230        /* temporary register for internal BPF JIT */
 231        [TMP_REG_1] = G1,
 232        [TMP_REG_2] = G2,
 233        [TMP_REG_3] = G3,
 234};
 235
 236static void emit(const u32 insn, struct jit_ctx *ctx)
 237{
 238        if (ctx->image != NULL)
 239                ctx->image[ctx->idx] = insn;
 240
 241        ctx->idx++;
 242}
 243
 244static void emit_call(u32 *func, struct jit_ctx *ctx)
 245{
 246        if (ctx->image != NULL) {
 247                void *here = &ctx->image[ctx->idx];
 248                unsigned int off;
 249
 250                off = (void *)func - here;
 251                ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff);
 252        }
 253        ctx->idx++;
 254}
 255
 256static void emit_nop(struct jit_ctx *ctx)
 257{
 258        emit(SETHI(0, G0), ctx);
 259}
 260
 261static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx)
 262{
 263        emit(OR | RS1(G0) | RS2(from) | RD(to), ctx);
 264}
 265
 266/* Emit 32-bit constant, zero extended. */
 267static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx)
 268{
 269        emit(SETHI(K, reg), ctx);
 270        emit(OR_LO(K, reg), ctx);
 271}
 272
 273/* Emit 32-bit constant, sign extended. */
 274static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx)
 275{
 276        if (K >= 0) {
 277                emit(SETHI(K, reg), ctx);
 278                emit(OR_LO(K, reg), ctx);
 279        } else {
 280                u32 hbits = ~(u32) K;
 281                u32 lbits = -0x400 | (u32) K;
 282
 283                emit(SETHI(hbits, reg), ctx);
 284                emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx);
 285        }
 286}
 287
 288static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx)
 289{
 290        emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx);
 291}
 292
 293static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx)
 294{
 295        emit(opcode | RS1(a) | RS2(b) | RD(c), ctx);
 296}
 297
 298static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm,
 299                       struct jit_ctx *ctx)
 300{
 301        bool small_immed = is_simm13(imm);
 302        unsigned int insn = opcode;
 303
 304        insn |= RS1(dst) | RD(dst);
 305        if (small_immed) {
 306                emit(insn | IMMED | S13(imm), ctx);
 307        } else {
 308                unsigned int tmp = bpf2sparc[TMP_REG_1];
 309
 310                ctx->tmp_1_used = true;
 311
 312                emit_set_const_sext(imm, tmp, ctx);
 313                emit(insn | RS2(tmp), ctx);
 314        }
 315}
 316
 317static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm,
 318                        unsigned int dst, struct jit_ctx *ctx)
 319{
 320        bool small_immed = is_simm13(imm);
 321        unsigned int insn = opcode;
 322
 323        insn |= RS1(src) | RD(dst);
 324        if (small_immed) {
 325                emit(insn | IMMED | S13(imm), ctx);
 326        } else {
 327                unsigned int tmp = bpf2sparc[TMP_REG_1];
 328
 329                ctx->tmp_1_used = true;
 330
 331                emit_set_const_sext(imm, tmp, ctx);
 332                emit(insn | RS2(tmp), ctx);
 333        }
 334}
 335
 336static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx)
 337{
 338        if (K >= 0 && is_simm13(K)) {
 339                /* or %g0, K, DEST */
 340                emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
 341        } else {
 342                emit_set_const(K, dest, ctx);
 343        }
 344}
 345
 346static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx)
 347{
 348        if (is_simm13(K)) {
 349                /* or %g0, K, DEST */
 350                emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
 351        } else {
 352                emit_set_const(K, dest, ctx);
 353        }
 354}
 355
 356static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx)
 357{
 358        if (is_simm13(K)) {
 359                /* or %g0, K, DEST */
 360                emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
 361        } else {
 362                emit_set_const_sext(K, dest, ctx);
 363        }
 364}
 365
 366static void analyze_64bit_constant(u32 high_bits, u32 low_bits,
 367                                   int *hbsp, int *lbsp, int *abbasp)
 368{
 369        int lowest_bit_set, highest_bit_set, all_bits_between_are_set;
 370        int i;
 371
 372        lowest_bit_set = highest_bit_set = -1;
 373        i = 0;
 374        do {
 375                if ((lowest_bit_set == -1) && ((low_bits >> i) & 1))
 376                        lowest_bit_set = i;
 377                if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1))
 378                        highest_bit_set = (64 - i - 1);
 379        }  while (++i < 32 && (highest_bit_set == -1 ||
 380                               lowest_bit_set == -1));
 381        if (i == 32) {
 382                i = 0;
 383                do {
 384                        if (lowest_bit_set == -1 && ((high_bits >> i) & 1))
 385                                lowest_bit_set = i + 32;
 386                        if (highest_bit_set == -1 &&
 387                            ((low_bits >> (32 - i - 1)) & 1))
 388                                highest_bit_set = 32 - i - 1;
 389                } while (++i < 32 && (highest_bit_set == -1 ||
 390                                      lowest_bit_set == -1));
 391        }
 392
 393        all_bits_between_are_set = 1;
 394        for (i = lowest_bit_set; i <= highest_bit_set; i++) {
 395                if (i < 32) {
 396                        if ((low_bits & (1 << i)) != 0)
 397                                continue;
 398                } else {
 399                        if ((high_bits & (1 << (i - 32))) != 0)
 400                                continue;
 401                }
 402                all_bits_between_are_set = 0;
 403                break;
 404        }
 405        *hbsp = highest_bit_set;
 406        *lbsp = lowest_bit_set;
 407        *abbasp = all_bits_between_are_set;
 408}
 409
 410static unsigned long create_simple_focus_bits(unsigned long high_bits,
 411                                              unsigned long low_bits,
 412                                              int lowest_bit_set, int shift)
 413{
 414        long hi, lo;
 415
 416        if (lowest_bit_set < 32) {
 417                lo = (low_bits >> lowest_bit_set) << shift;
 418                hi = ((high_bits << (32 - lowest_bit_set)) << shift);
 419        } else {
 420                lo = 0;
 421                hi = ((high_bits >> (lowest_bit_set - 32)) << shift);
 422        }
 423        return hi | lo;
 424}
 425
 426static bool const64_is_2insns(unsigned long high_bits,
 427                              unsigned long low_bits)
 428{
 429        int highest_bit_set, lowest_bit_set, all_bits_between_are_set;
 430
 431        if (high_bits == 0 || high_bits == 0xffffffff)
 432                return true;
 433
 434        analyze_64bit_constant(high_bits, low_bits,
 435                               &highest_bit_set, &lowest_bit_set,
 436                               &all_bits_between_are_set);
 437
 438        if ((highest_bit_set == 63 || lowest_bit_set == 0) &&
 439            all_bits_between_are_set != 0)
 440                return true;
 441
 442        if (highest_bit_set - lowest_bit_set < 21)
 443                return true;
 444
 445        return false;
 446}
 447
 448static void sparc_emit_set_const64_quick2(unsigned long high_bits,
 449                                          unsigned long low_imm,
 450                                          unsigned int dest,
 451                                          int shift_count, struct jit_ctx *ctx)
 452{
 453        emit_loadimm32(high_bits, dest, ctx);
 454
 455        /* Now shift it up into place.  */
 456        emit_alu_K(SLLX, dest, shift_count, ctx);
 457
 458        /* If there is a low immediate part piece, finish up by
 459         * putting that in as well.
 460         */
 461        if (low_imm != 0)
 462                emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx);
 463}
 464
 465static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx)
 466{
 467        int all_bits_between_are_set, lowest_bit_set, highest_bit_set;
 468        unsigned int tmp = bpf2sparc[TMP_REG_1];
 469        u32 low_bits = (K & 0xffffffff);
 470        u32 high_bits = (K >> 32);
 471
 472        /* These two tests also take care of all of the one
 473         * instruction cases.
 474         */
 475        if (high_bits == 0xffffffff && (low_bits & 0x80000000))
 476                return emit_loadimm_sext(K, dest, ctx);
 477        if (high_bits == 0x00000000)
 478                return emit_loadimm32(K, dest, ctx);
 479
 480        analyze_64bit_constant(high_bits, low_bits, &highest_bit_set,
 481                               &lowest_bit_set, &all_bits_between_are_set);
 482
 483        /* 1) mov       -1, %reg
 484         *    sllx      %reg, shift, %reg
 485         * 2) mov       -1, %reg
 486         *    srlx      %reg, shift, %reg
 487         * 3) mov       some_small_const, %reg
 488         *    sllx      %reg, shift, %reg
 489         */
 490        if (((highest_bit_set == 63 || lowest_bit_set == 0) &&
 491             all_bits_between_are_set != 0) ||
 492            ((highest_bit_set - lowest_bit_set) < 12)) {
 493                int shift = lowest_bit_set;
 494                long the_const = -1;
 495
 496                if ((highest_bit_set != 63 && lowest_bit_set != 0) ||
 497                    all_bits_between_are_set == 0) {
 498                        the_const =
 499                                create_simple_focus_bits(high_bits, low_bits,
 500                                                         lowest_bit_set, 0);
 501                } else if (lowest_bit_set == 0)
 502                        shift = -(63 - highest_bit_set);
 503
 504                emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx);
 505                if (shift > 0)
 506                        emit_alu_K(SLLX, dest, shift, ctx);
 507                else if (shift < 0)
 508                        emit_alu_K(SRLX, dest, -shift, ctx);
 509
 510                return;
 511        }
 512
 513        /* Now a range of 22 or less bits set somewhere.
 514         * 1) sethi     %hi(focus_bits), %reg
 515         *    sllx      %reg, shift, %reg
 516         * 2) sethi     %hi(focus_bits), %reg
 517         *    srlx      %reg, shift, %reg
 518         */
 519        if ((highest_bit_set - lowest_bit_set) < 21) {
 520                unsigned long focus_bits =
 521                        create_simple_focus_bits(high_bits, low_bits,
 522                                                 lowest_bit_set, 10);
 523
 524                emit(SETHI(focus_bits, dest), ctx);
 525
 526                /* If lowest_bit_set == 10 then a sethi alone could
 527                 * have done it.
 528                 */
 529                if (lowest_bit_set < 10)
 530                        emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx);
 531                else if (lowest_bit_set > 10)
 532                        emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx);
 533                return;
 534        }
 535
 536        /* Ok, now 3 instruction sequences.  */
 537        if (low_bits == 0) {
 538                emit_loadimm32(high_bits, dest, ctx);
 539                emit_alu_K(SLLX, dest, 32, ctx);
 540                return;
 541        }
 542
 543        /* We may be able to do something quick
 544         * when the constant is negated, so try that.
 545         */
 546        if (const64_is_2insns((~high_bits) & 0xffffffff,
 547                              (~low_bits) & 0xfffffc00)) {
 548                /* NOTE: The trailing bits get XOR'd so we need the
 549                 * non-negated bits, not the negated ones.
 550                 */
 551                unsigned long trailing_bits = low_bits & 0x3ff;
 552
 553                if ((((~high_bits) & 0xffffffff) == 0 &&
 554                     ((~low_bits) & 0x80000000) == 0) ||
 555                    (((~high_bits) & 0xffffffff) == 0xffffffff &&
 556                     ((~low_bits) & 0x80000000) != 0)) {
 557                        unsigned long fast_int = (~low_bits & 0xffffffff);
 558
 559                        if ((is_sethi(fast_int) &&
 560                             (~high_bits & 0xffffffff) == 0)) {
 561                                emit(SETHI(fast_int, dest), ctx);
 562                        } else if (is_simm13(fast_int)) {
 563                                emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx);
 564                        } else {
 565                                emit_loadimm64(fast_int, dest, ctx);
 566                        }
 567                } else {
 568                        u64 n = ((~low_bits) & 0xfffffc00) |
 569                                (((unsigned long)((~high_bits) & 0xffffffff))<<32);
 570                        emit_loadimm64(n, dest, ctx);
 571                }
 572
 573                low_bits = -0x400 | trailing_bits;
 574
 575                emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx);
 576                return;
 577        }
 578
 579        /* 1) sethi     %hi(xxx), %reg
 580         *    or        %reg, %lo(xxx), %reg
 581         *    sllx      %reg, yyy, %reg
 582         */
 583        if ((highest_bit_set - lowest_bit_set) < 32) {
 584                unsigned long focus_bits =
 585                        create_simple_focus_bits(high_bits, low_bits,
 586                                                 lowest_bit_set, 0);
 587
 588                /* So what we know is that the set bits straddle the
 589                 * middle of the 64-bit word.
 590                 */
 591                sparc_emit_set_const64_quick2(focus_bits, 0, dest,
 592                                              lowest_bit_set, ctx);
 593                return;
 594        }
 595
 596        /* 1) sethi     %hi(high_bits), %reg
 597         *    or        %reg, %lo(high_bits), %reg
 598         *    sllx      %reg, 32, %reg
 599         *    or        %reg, low_bits, %reg
 600         */
 601        if (is_simm13(low_bits) && ((int)low_bits > 0)) {
 602                sparc_emit_set_const64_quick2(high_bits, low_bits,
 603                                              dest, 32, ctx);
 604                return;
 605        }
 606
 607        /* Oh well, we tried... Do a full 64-bit decomposition.  */
 608        ctx->tmp_1_used = true;
 609
 610        emit_loadimm32(high_bits, tmp, ctx);
 611        emit_loadimm32(low_bits, dest, ctx);
 612        emit_alu_K(SLLX, tmp, 32, ctx);
 613        emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx);
 614}
 615
 616static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx,
 617                        struct jit_ctx *ctx)
 618{
 619        unsigned int off = to_idx - from_idx;
 620
 621        if (br_opc & XCC)
 622                emit(br_opc | WDISP19(off << 2), ctx);
 623        else
 624                emit(br_opc | WDISP22(off << 2), ctx);
 625}
 626
 627static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
 628                        const u8 dst, const u8 src, struct jit_ctx *ctx)
 629{
 630        unsigned int off = to_idx - from_idx;
 631
 632        emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx);
 633}
 634
 635static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
 636                         const u8 dst, s32 imm, struct jit_ctx *ctx)
 637{
 638        unsigned int off = to_idx - from_idx;
 639
 640        emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx);
 641}
 642
 643#define emit_read_y(REG, CTX)   emit(RD_Y | RD(REG), CTX)
 644#define emit_write_y(REG, CTX)  emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX)
 645
 646#define emit_cmp(R1, R2, CTX)                           \
 647        emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
 648
 649#define emit_cmpi(R1, IMM, CTX)                         \
 650        emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
 651
 652#define emit_btst(R1, R2, CTX)                          \
 653        emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
 654
 655#define emit_btsti(R1, IMM, CTX)                        \
 656        emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
 657
 658static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
 659                                   const s32 imm, bool is_imm, int branch_dst,
 660                                   struct jit_ctx *ctx)
 661{
 662        bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0;
 663        const u8 tmp = bpf2sparc[TMP_REG_1];
 664
 665        branch_dst = ctx->offset[branch_dst];
 666
 667        if (!is_simm10(branch_dst - ctx->idx) ||
 668            BPF_OP(code) == BPF_JSET)
 669                use_cbcond = false;
 670
 671        if (is_imm) {
 672                bool fits = true;
 673
 674                if (use_cbcond) {
 675                        if (!is_simm5(imm))
 676                                fits = false;
 677                } else if (!is_simm13(imm)) {
 678                        fits = false;
 679                }
 680                if (!fits) {
 681                        ctx->tmp_1_used = true;
 682                        emit_loadimm_sext(imm, tmp, ctx);
 683                        src = tmp;
 684                        is_imm = false;
 685                }
 686        }
 687
 688        if (!use_cbcond) {
 689                u32 br_opcode;
 690
 691                if (BPF_OP(code) == BPF_JSET) {
 692                        if (is_imm)
 693                                emit_btsti(dst, imm, ctx);
 694                        else
 695                                emit_btst(dst, src, ctx);
 696                } else {
 697                        if (is_imm)
 698                                emit_cmpi(dst, imm, ctx);
 699                        else
 700                                emit_cmp(dst, src, ctx);
 701                }
 702                switch (BPF_OP(code)) {
 703                case BPF_JEQ:
 704                        br_opcode = BE;
 705                        break;
 706                case BPF_JGT:
 707                        br_opcode = BGU;
 708                        break;
 709                case BPF_JLT:
 710                        br_opcode = BLU;
 711                        break;
 712                case BPF_JGE:
 713                        br_opcode = BGEU;
 714                        break;
 715                case BPF_JLE:
 716                        br_opcode = BLEU;
 717                        break;
 718                case BPF_JSET:
 719                case BPF_JNE:
 720                        br_opcode = BNE;
 721                        break;
 722                case BPF_JSGT:
 723                        br_opcode = BG;
 724                        break;
 725                case BPF_JSLT:
 726                        br_opcode = BL;
 727                        break;
 728                case BPF_JSGE:
 729                        br_opcode = BGE;
 730                        break;
 731                case BPF_JSLE:
 732                        br_opcode = BLE;
 733                        break;
 734                default:
 735                        /* Make sure we dont leak kernel information to the
 736                         * user.
 737                         */
 738                        return -EFAULT;
 739                }
 740                emit_branch(br_opcode, ctx->idx, branch_dst, ctx);
 741                emit_nop(ctx);
 742        } else {
 743                u32 cbcond_opcode;
 744
 745                switch (BPF_OP(code)) {
 746                case BPF_JEQ:
 747                        cbcond_opcode = CBCONDE;
 748                        break;
 749                case BPF_JGT:
 750                        cbcond_opcode = CBCONDGU;
 751                        break;
 752                case BPF_JLT:
 753                        cbcond_opcode = CBCONDLU;
 754                        break;
 755                case BPF_JGE:
 756                        cbcond_opcode = CBCONDGEU;
 757                        break;
 758                case BPF_JLE:
 759                        cbcond_opcode = CBCONDLEU;
 760                        break;
 761                case BPF_JNE:
 762                        cbcond_opcode = CBCONDNE;
 763                        break;
 764                case BPF_JSGT:
 765                        cbcond_opcode = CBCONDG;
 766                        break;
 767                case BPF_JSLT:
 768                        cbcond_opcode = CBCONDL;
 769                        break;
 770                case BPF_JSGE:
 771                        cbcond_opcode = CBCONDGE;
 772                        break;
 773                case BPF_JSLE:
 774                        cbcond_opcode = CBCONDLE;
 775                        break;
 776                default:
 777                        /* Make sure we dont leak kernel information to the
 778                         * user.
 779                         */
 780                        return -EFAULT;
 781                }
 782                cbcond_opcode |= CBCOND_OP;
 783                if (is_imm)
 784                        emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst,
 785                                     dst, imm, ctx);
 786                else
 787                        emit_cbcond(cbcond_opcode, ctx->idx, branch_dst,
 788                                    dst, src, ctx);
 789        }
 790        return 0;
 791}
 792
 793/* Just skip the save instruction and the ctx register move.  */
 794#define BPF_TAILCALL_PROLOGUE_SKIP      32
 795#define BPF_TAILCALL_CNT_SP_OFF         (STACK_BIAS + 128)
 796
 797static void build_prologue(struct jit_ctx *ctx)
 798{
 799        s32 stack_needed = BASE_STACKFRAME;
 800
 801        if (ctx->saw_frame_pointer || ctx->saw_tail_call) {
 802                struct bpf_prog *prog = ctx->prog;
 803                u32 stack_depth;
 804
 805                stack_depth = prog->aux->stack_depth;
 806                stack_needed += round_up(stack_depth, 16);
 807        }
 808
 809        if (ctx->saw_tail_call)
 810                stack_needed += 8;
 811
 812        /* save %sp, -176, %sp */
 813        emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx);
 814
 815        /* tail_call_cnt = 0 */
 816        if (ctx->saw_tail_call) {
 817                u32 off = BPF_TAILCALL_CNT_SP_OFF;
 818
 819                emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx);
 820        } else {
 821                emit_nop(ctx);
 822        }
 823        if (ctx->saw_frame_pointer) {
 824                const u8 vfp = bpf2sparc[BPF_REG_FP];
 825
 826                emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx);
 827        } else {
 828                emit_nop(ctx);
 829        }
 830
 831        emit_reg_move(I0, O0, ctx);
 832        emit_reg_move(I1, O1, ctx);
 833        emit_reg_move(I2, O2, ctx);
 834        emit_reg_move(I3, O3, ctx);
 835        emit_reg_move(I4, O4, ctx);
 836        /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
 837}
 838
 839static void build_epilogue(struct jit_ctx *ctx)
 840{
 841        ctx->epilogue_offset = ctx->idx;
 842
 843        /* ret (jmpl %i7 + 8, %g0) */
 844        emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx);
 845
 846        /* restore %i5, %g0, %o0 */
 847        emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx);
 848}
 849
 850static void emit_tail_call(struct jit_ctx *ctx)
 851{
 852        const u8 bpf_array = bpf2sparc[BPF_REG_2];
 853        const u8 bpf_index = bpf2sparc[BPF_REG_3];
 854        const u8 tmp = bpf2sparc[TMP_REG_1];
 855        u32 off;
 856
 857        ctx->saw_tail_call = true;
 858
 859        off = offsetof(struct bpf_array, map.max_entries);
 860        emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
 861        emit_cmp(bpf_index, tmp, ctx);
 862#define OFFSET1 17
 863        emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx);
 864        emit_nop(ctx);
 865
 866        off = BPF_TAILCALL_CNT_SP_OFF;
 867        emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
 868        emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx);
 869#define OFFSET2 13
 870        emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx);
 871        emit_nop(ctx);
 872
 873        emit_alu_K(ADD, tmp, 1, ctx);
 874        off = BPF_TAILCALL_CNT_SP_OFF;
 875        emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
 876
 877        emit_alu3_K(SLL, bpf_index, 3, tmp, ctx);
 878        emit_alu(ADD, bpf_array, tmp, ctx);
 879        off = offsetof(struct bpf_array, ptrs);
 880        emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
 881
 882        emit_cmpi(tmp, 0, ctx);
 883#define OFFSET3 5
 884        emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx);
 885        emit_nop(ctx);
 886
 887        off = offsetof(struct bpf_prog, bpf_func);
 888        emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
 889
 890        off = BPF_TAILCALL_PROLOGUE_SKIP;
 891        emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx);
 892        emit_nop(ctx);
 893}
 894
 895static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 896{
 897        const u8 code = insn->code;
 898        const u8 dst = bpf2sparc[insn->dst_reg];
 899        const u8 src = bpf2sparc[insn->src_reg];
 900        const int i = insn - ctx->prog->insnsi;
 901        const s16 off = insn->off;
 902        const s32 imm = insn->imm;
 903
 904        if (insn->src_reg == BPF_REG_FP)
 905                ctx->saw_frame_pointer = true;
 906
 907        switch (code) {
 908        /* dst = src */
 909        case BPF_ALU | BPF_MOV | BPF_X:
 910                emit_alu3_K(SRL, src, 0, dst, ctx);
 911                if (insn_is_zext(&insn[1]))
 912                        return 1;
 913                break;
 914        case BPF_ALU64 | BPF_MOV | BPF_X:
 915                emit_reg_move(src, dst, ctx);
 916                break;
 917        /* dst = dst OP src */
 918        case BPF_ALU | BPF_ADD | BPF_X:
 919        case BPF_ALU64 | BPF_ADD | BPF_X:
 920                emit_alu(ADD, src, dst, ctx);
 921                goto do_alu32_trunc;
 922        case BPF_ALU | BPF_SUB | BPF_X:
 923        case BPF_ALU64 | BPF_SUB | BPF_X:
 924                emit_alu(SUB, src, dst, ctx);
 925                goto do_alu32_trunc;
 926        case BPF_ALU | BPF_AND | BPF_X:
 927        case BPF_ALU64 | BPF_AND | BPF_X:
 928                emit_alu(AND, src, dst, ctx);
 929                goto do_alu32_trunc;
 930        case BPF_ALU | BPF_OR | BPF_X:
 931        case BPF_ALU64 | BPF_OR | BPF_X:
 932                emit_alu(OR, src, dst, ctx);
 933                goto do_alu32_trunc;
 934        case BPF_ALU | BPF_XOR | BPF_X:
 935        case BPF_ALU64 | BPF_XOR | BPF_X:
 936                emit_alu(XOR, src, dst, ctx);
 937                goto do_alu32_trunc;
 938        case BPF_ALU | BPF_MUL | BPF_X:
 939                emit_alu(MUL, src, dst, ctx);
 940                goto do_alu32_trunc;
 941        case BPF_ALU64 | BPF_MUL | BPF_X:
 942                emit_alu(MULX, src, dst, ctx);
 943                break;
 944        case BPF_ALU | BPF_DIV | BPF_X:
 945                emit_write_y(G0, ctx);
 946                emit_alu(DIV, src, dst, ctx);
 947                if (insn_is_zext(&insn[1]))
 948                        return 1;
 949                break;
 950        case BPF_ALU64 | BPF_DIV | BPF_X:
 951                emit_alu(UDIVX, src, dst, ctx);
 952                break;
 953        case BPF_ALU | BPF_MOD | BPF_X: {
 954                const u8 tmp = bpf2sparc[TMP_REG_1];
 955
 956                ctx->tmp_1_used = true;
 957
 958                emit_write_y(G0, ctx);
 959                emit_alu3(DIV, dst, src, tmp, ctx);
 960                emit_alu3(MULX, tmp, src, tmp, ctx);
 961                emit_alu3(SUB, dst, tmp, dst, ctx);
 962                goto do_alu32_trunc;
 963        }
 964        case BPF_ALU64 | BPF_MOD | BPF_X: {
 965                const u8 tmp = bpf2sparc[TMP_REG_1];
 966
 967                ctx->tmp_1_used = true;
 968
 969                emit_alu3(UDIVX, dst, src, tmp, ctx);
 970                emit_alu3(MULX, tmp, src, tmp, ctx);
 971                emit_alu3(SUB, dst, tmp, dst, ctx);
 972                break;
 973        }
 974        case BPF_ALU | BPF_LSH | BPF_X:
 975                emit_alu(SLL, src, dst, ctx);
 976                goto do_alu32_trunc;
 977        case BPF_ALU64 | BPF_LSH | BPF_X:
 978                emit_alu(SLLX, src, dst, ctx);
 979                break;
 980        case BPF_ALU | BPF_RSH | BPF_X:
 981                emit_alu(SRL, src, dst, ctx);
 982                if (insn_is_zext(&insn[1]))
 983                        return 1;
 984                break;
 985        case BPF_ALU64 | BPF_RSH | BPF_X:
 986                emit_alu(SRLX, src, dst, ctx);
 987                break;
 988        case BPF_ALU | BPF_ARSH | BPF_X:
 989                emit_alu(SRA, src, dst, ctx);
 990                goto do_alu32_trunc;
 991        case BPF_ALU64 | BPF_ARSH | BPF_X:
 992                emit_alu(SRAX, src, dst, ctx);
 993                break;
 994
 995        /* dst = -dst */
 996        case BPF_ALU | BPF_NEG:
 997        case BPF_ALU64 | BPF_NEG:
 998                emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx);
 999                goto do_alu32_trunc;
1000
1001        case BPF_ALU | BPF_END | BPF_FROM_BE:
1002                switch (imm) {
1003                case 16:
1004                        emit_alu_K(SLL, dst, 16, ctx);
1005                        emit_alu_K(SRL, dst, 16, ctx);
1006                        if (insn_is_zext(&insn[1]))
1007                                return 1;
1008                        break;
1009                case 32:
1010                        if (!ctx->prog->aux->verifier_zext)
1011                                emit_alu_K(SRL, dst, 0, ctx);
1012                        break;
1013                case 64:
1014                        /* nop */
1015                        break;
1016
1017                }
1018                break;
1019
1020        /* dst = BSWAP##imm(dst) */
1021        case BPF_ALU | BPF_END | BPF_FROM_LE: {
1022                const u8 tmp = bpf2sparc[TMP_REG_1];
1023                const u8 tmp2 = bpf2sparc[TMP_REG_2];
1024
1025                ctx->tmp_1_used = true;
1026                switch (imm) {
1027                case 16:
1028                        emit_alu3_K(AND, dst, 0xff, tmp, ctx);
1029                        emit_alu3_K(SRL, dst, 8, dst, ctx);
1030                        emit_alu3_K(AND, dst, 0xff, dst, ctx);
1031                        emit_alu3_K(SLL, tmp, 8, tmp, ctx);
1032                        emit_alu(OR, tmp, dst, ctx);
1033                        if (insn_is_zext(&insn[1]))
1034                                return 1;
1035                        break;
1036
1037                case 32:
1038                        ctx->tmp_2_used = true;
1039                        emit_alu3_K(SRL, dst, 24, tmp, ctx);    /* tmp  = dst >> 24 */
1040                        emit_alu3_K(SRL, dst, 16, tmp2, ctx);   /* tmp2 = dst >> 16 */
1041                        emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1042                        emit_alu3_K(SLL, tmp2, 8, tmp2, ctx);   /* tmp2 = tmp2 << 8 */
1043                        emit_alu(OR, tmp2, tmp, ctx);           /* tmp  = tmp | tmp2 */
1044                        emit_alu3_K(SRL, dst, 8, tmp2, ctx);    /* tmp2 = dst >> 8 */
1045                        emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
1046                        emit_alu3_K(SLL, tmp2, 16, tmp2, ctx);  /* tmp2 = tmp2 << 16 */
1047                        emit_alu(OR, tmp2, tmp, ctx);           /* tmp  = tmp | tmp2 */
1048                        emit_alu3_K(AND, dst, 0xff, dst, ctx);  /* dst  = dst & 0xff */
1049                        emit_alu3_K(SLL, dst, 24, dst, ctx);    /* dst  = dst << 24 */
1050                        emit_alu(OR, tmp, dst, ctx);            /* dst  = dst | tmp */
1051                        if (insn_is_zext(&insn[1]))
1052                                return 1;
1053                        break;
1054
1055                case 64:
1056                        emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx);
1057                        emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1058                        emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx);
1059                        break;
1060                }
1061                break;
1062        }
1063        /* dst = imm */
1064        case BPF_ALU | BPF_MOV | BPF_K:
1065                emit_loadimm32(imm, dst, ctx);
1066                if (insn_is_zext(&insn[1]))
1067                        return 1;
1068                break;
1069        case BPF_ALU64 | BPF_MOV | BPF_K:
1070                emit_loadimm_sext(imm, dst, ctx);
1071                break;
1072        /* dst = dst OP imm */
1073        case BPF_ALU | BPF_ADD | BPF_K:
1074        case BPF_ALU64 | BPF_ADD | BPF_K:
1075                emit_alu_K(ADD, dst, imm, ctx);
1076                goto do_alu32_trunc;
1077        case BPF_ALU | BPF_SUB | BPF_K:
1078        case BPF_ALU64 | BPF_SUB | BPF_K:
1079                emit_alu_K(SUB, dst, imm, ctx);
1080                goto do_alu32_trunc;
1081        case BPF_ALU | BPF_AND | BPF_K:
1082        case BPF_ALU64 | BPF_AND | BPF_K:
1083                emit_alu_K(AND, dst, imm, ctx);
1084                goto do_alu32_trunc;
1085        case BPF_ALU | BPF_OR | BPF_K:
1086        case BPF_ALU64 | BPF_OR | BPF_K:
1087                emit_alu_K(OR, dst, imm, ctx);
1088                goto do_alu32_trunc;
1089        case BPF_ALU | BPF_XOR | BPF_K:
1090        case BPF_ALU64 | BPF_XOR | BPF_K:
1091                emit_alu_K(XOR, dst, imm, ctx);
1092                goto do_alu32_trunc;
1093        case BPF_ALU | BPF_MUL | BPF_K:
1094                emit_alu_K(MUL, dst, imm, ctx);
1095                goto do_alu32_trunc;
1096        case BPF_ALU64 | BPF_MUL | BPF_K:
1097                emit_alu_K(MULX, dst, imm, ctx);
1098                break;
1099        case BPF_ALU | BPF_DIV | BPF_K:
1100                if (imm == 0)
1101                        return -EINVAL;
1102
1103                emit_write_y(G0, ctx);
1104                emit_alu_K(DIV, dst, imm, ctx);
1105                goto do_alu32_trunc;
1106        case BPF_ALU64 | BPF_DIV | BPF_K:
1107                if (imm == 0)
1108                        return -EINVAL;
1109
1110                emit_alu_K(UDIVX, dst, imm, ctx);
1111                break;
1112        case BPF_ALU64 | BPF_MOD | BPF_K:
1113        case BPF_ALU | BPF_MOD | BPF_K: {
1114                const u8 tmp = bpf2sparc[TMP_REG_2];
1115                unsigned int div;
1116
1117                if (imm == 0)
1118                        return -EINVAL;
1119
1120                div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
1121
1122                ctx->tmp_2_used = true;
1123
1124                if (BPF_CLASS(code) != BPF_ALU64)
1125                        emit_write_y(G0, ctx);
1126                if (is_simm13(imm)) {
1127                        emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
1128                        emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);
1129                        emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1130                } else {
1131                        const u8 tmp1 = bpf2sparc[TMP_REG_1];
1132
1133                        ctx->tmp_1_used = true;
1134
1135                        emit_set_const_sext(imm, tmp1, ctx);
1136                        emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx);
1137                        emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx);
1138                        emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
1139                }
1140                goto do_alu32_trunc;
1141        }
1142        case BPF_ALU | BPF_LSH | BPF_K:
1143                emit_alu_K(SLL, dst, imm, ctx);
1144                goto do_alu32_trunc;
1145        case BPF_ALU64 | BPF_LSH | BPF_K:
1146                emit_alu_K(SLLX, dst, imm, ctx);
1147                break;
1148        case BPF_ALU | BPF_RSH | BPF_K:
1149                emit_alu_K(SRL, dst, imm, ctx);
1150                if (insn_is_zext(&insn[1]))
1151                        return 1;
1152                break;
1153        case BPF_ALU64 | BPF_RSH | BPF_K:
1154                emit_alu_K(SRLX, dst, imm, ctx);
1155                break;
1156        case BPF_ALU | BPF_ARSH | BPF_K:
1157                emit_alu_K(SRA, dst, imm, ctx);
1158                goto do_alu32_trunc;
1159        case BPF_ALU64 | BPF_ARSH | BPF_K:
1160                emit_alu_K(SRAX, dst, imm, ctx);
1161                break;
1162
1163        do_alu32_trunc:
1164                if (BPF_CLASS(code) == BPF_ALU &&
1165                    !ctx->prog->aux->verifier_zext)
1166                        emit_alu_K(SRL, dst, 0, ctx);
1167                break;
1168
1169        /* JUMP off */
1170        case BPF_JMP | BPF_JA:
1171                emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx);
1172                emit_nop(ctx);
1173                break;
1174        /* IF (dst COND src) JUMP off */
1175        case BPF_JMP | BPF_JEQ | BPF_X:
1176        case BPF_JMP | BPF_JGT | BPF_X:
1177        case BPF_JMP | BPF_JLT | BPF_X:
1178        case BPF_JMP | BPF_JGE | BPF_X:
1179        case BPF_JMP | BPF_JLE | BPF_X:
1180        case BPF_JMP | BPF_JNE | BPF_X:
1181        case BPF_JMP | BPF_JSGT | BPF_X:
1182        case BPF_JMP | BPF_JSLT | BPF_X:
1183        case BPF_JMP | BPF_JSGE | BPF_X:
1184        case BPF_JMP | BPF_JSLE | BPF_X:
1185        case BPF_JMP | BPF_JSET | BPF_X: {
1186                int err;
1187
1188                err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx);
1189                if (err)
1190                        return err;
1191                break;
1192        }
1193        /* IF (dst COND imm) JUMP off */
1194        case BPF_JMP | BPF_JEQ | BPF_K:
1195        case BPF_JMP | BPF_JGT | BPF_K:
1196        case BPF_JMP | BPF_JLT | BPF_K:
1197        case BPF_JMP | BPF_JGE | BPF_K:
1198        case BPF_JMP | BPF_JLE | BPF_K:
1199        case BPF_JMP | BPF_JNE | BPF_K:
1200        case BPF_JMP | BPF_JSGT | BPF_K:
1201        case BPF_JMP | BPF_JSLT | BPF_K:
1202        case BPF_JMP | BPF_JSGE | BPF_K:
1203        case BPF_JMP | BPF_JSLE | BPF_K:
1204        case BPF_JMP | BPF_JSET | BPF_K: {
1205                int err;
1206
1207                err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx);
1208                if (err)
1209                        return err;
1210                break;
1211        }
1212
1213        /* function call */
1214        case BPF_JMP | BPF_CALL:
1215        {
1216                u8 *func = ((u8 *)__bpf_call_base) + imm;
1217
1218                ctx->saw_call = true;
1219
1220                emit_call((u32 *)func, ctx);
1221                emit_nop(ctx);
1222
1223                emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
1224                break;
1225        }
1226
1227        /* tail call */
1228        case BPF_JMP | BPF_TAIL_CALL:
1229                emit_tail_call(ctx);
1230                break;
1231
1232        /* function return */
1233        case BPF_JMP | BPF_EXIT:
1234                /* Optimization: when last instruction is EXIT,
1235                   simply fallthrough to epilogue. */
1236                if (i == ctx->prog->len - 1)
1237                        break;
1238                emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx);
1239                emit_nop(ctx);
1240                break;
1241
1242        /* dst = imm64 */
1243        case BPF_LD | BPF_IMM | BPF_DW:
1244        {
1245                const struct bpf_insn insn1 = insn[1];
1246                u64 imm64;
1247
1248                imm64 = (u64)insn1.imm << 32 | (u32)imm;
1249                emit_loadimm64(imm64, dst, ctx);
1250
1251                return 1;
1252        }
1253
1254        /* LDX: dst = *(size *)(src + off) */
1255        case BPF_LDX | BPF_MEM | BPF_W:
1256        case BPF_LDX | BPF_MEM | BPF_H:
1257        case BPF_LDX | BPF_MEM | BPF_B:
1258        case BPF_LDX | BPF_MEM | BPF_DW: {
1259                const u8 tmp = bpf2sparc[TMP_REG_1];
1260                u32 opcode = 0, rs2;
1261
1262                ctx->tmp_1_used = true;
1263                switch (BPF_SIZE(code)) {
1264                case BPF_W:
1265                        opcode = LD32;
1266                        break;
1267                case BPF_H:
1268                        opcode = LD16;
1269                        break;
1270                case BPF_B:
1271                        opcode = LD8;
1272                        break;
1273                case BPF_DW:
1274                        opcode = LD64;
1275                        break;
1276                }
1277
1278                if (is_simm13(off)) {
1279                        opcode |= IMMED;
1280                        rs2 = S13(off);
1281                } else {
1282                        emit_loadimm(off, tmp, ctx);
1283                        rs2 = RS2(tmp);
1284                }
1285                emit(opcode | RS1(src) | rs2 | RD(dst), ctx);
1286                if (opcode != LD64 && insn_is_zext(&insn[1]))
1287                        return 1;
1288                break;
1289        }
1290        /* ST: *(size *)(dst + off) = imm */
1291        case BPF_ST | BPF_MEM | BPF_W:
1292        case BPF_ST | BPF_MEM | BPF_H:
1293        case BPF_ST | BPF_MEM | BPF_B:
1294        case BPF_ST | BPF_MEM | BPF_DW: {
1295                const u8 tmp = bpf2sparc[TMP_REG_1];
1296                const u8 tmp2 = bpf2sparc[TMP_REG_2];
1297                u32 opcode = 0, rs2;
1298
1299                if (insn->dst_reg == BPF_REG_FP)
1300                        ctx->saw_frame_pointer = true;
1301
1302                ctx->tmp_2_used = true;
1303                emit_loadimm(imm, tmp2, ctx);
1304
1305                switch (BPF_SIZE(code)) {
1306                case BPF_W:
1307                        opcode = ST32;
1308                        break;
1309                case BPF_H:
1310                        opcode = ST16;
1311                        break;
1312                case BPF_B:
1313                        opcode = ST8;
1314                        break;
1315                case BPF_DW:
1316                        opcode = ST64;
1317                        break;
1318                }
1319
1320                if (is_simm13(off)) {
1321                        opcode |= IMMED;
1322                        rs2 = S13(off);
1323                } else {
1324                        ctx->tmp_1_used = true;
1325                        emit_loadimm(off, tmp, ctx);
1326                        rs2 = RS2(tmp);
1327                }
1328                emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx);
1329                break;
1330        }
1331
1332        /* STX: *(size *)(dst + off) = src */
1333        case BPF_STX | BPF_MEM | BPF_W:
1334        case BPF_STX | BPF_MEM | BPF_H:
1335        case BPF_STX | BPF_MEM | BPF_B:
1336        case BPF_STX | BPF_MEM | BPF_DW: {
1337                const u8 tmp = bpf2sparc[TMP_REG_1];
1338                u32 opcode = 0, rs2;
1339
1340                if (insn->dst_reg == BPF_REG_FP)
1341                        ctx->saw_frame_pointer = true;
1342
1343                switch (BPF_SIZE(code)) {
1344                case BPF_W:
1345                        opcode = ST32;
1346                        break;
1347                case BPF_H:
1348                        opcode = ST16;
1349                        break;
1350                case BPF_B:
1351                        opcode = ST8;
1352                        break;
1353                case BPF_DW:
1354                        opcode = ST64;
1355                        break;
1356                }
1357                if (is_simm13(off)) {
1358                        opcode |= IMMED;
1359                        rs2 = S13(off);
1360                } else {
1361                        ctx->tmp_1_used = true;
1362                        emit_loadimm(off, tmp, ctx);
1363                        rs2 = RS2(tmp);
1364                }
1365                emit(opcode | RS1(dst) | rs2 | RD(src), ctx);
1366                break;
1367        }
1368
1369        case BPF_STX | BPF_ATOMIC | BPF_W: {
1370                const u8 tmp = bpf2sparc[TMP_REG_1];
1371                const u8 tmp2 = bpf2sparc[TMP_REG_2];
1372                const u8 tmp3 = bpf2sparc[TMP_REG_3];
1373
1374                if (insn->imm != BPF_ADD) {
1375                        pr_err_once("unknown atomic op %02x\n", insn->imm);
1376                        return -EINVAL;
1377                }
1378
1379                /* lock *(u32 *)(dst + off) += src */
1380
1381                if (insn->dst_reg == BPF_REG_FP)
1382                        ctx->saw_frame_pointer = true;
1383
1384                ctx->tmp_1_used = true;
1385                ctx->tmp_2_used = true;
1386                ctx->tmp_3_used = true;
1387                emit_loadimm(off, tmp, ctx);
1388                emit_alu3(ADD, dst, tmp, tmp, ctx);
1389
1390                emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1391                emit_alu3(ADD, tmp2, src, tmp3, ctx);
1392                emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1393                emit_cmp(tmp2, tmp3, ctx);
1394                emit_branch(BNE, 4, 0, ctx);
1395                emit_nop(ctx);
1396                break;
1397        }
1398        /* STX XADD: lock *(u64 *)(dst + off) += src */
1399        case BPF_STX | BPF_ATOMIC | BPF_DW: {
1400                const u8 tmp = bpf2sparc[TMP_REG_1];
1401                const u8 tmp2 = bpf2sparc[TMP_REG_2];
1402                const u8 tmp3 = bpf2sparc[TMP_REG_3];
1403
1404                if (insn->imm != BPF_ADD) {
1405                        pr_err_once("unknown atomic op %02x\n", insn->imm);
1406                        return -EINVAL;
1407                }
1408
1409                if (insn->dst_reg == BPF_REG_FP)
1410                        ctx->saw_frame_pointer = true;
1411
1412                ctx->tmp_1_used = true;
1413                ctx->tmp_2_used = true;
1414                ctx->tmp_3_used = true;
1415                emit_loadimm(off, tmp, ctx);
1416                emit_alu3(ADD, dst, tmp, tmp, ctx);
1417
1418                emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
1419                emit_alu3(ADD, tmp2, src, tmp3, ctx);
1420                emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
1421                emit_cmp(tmp2, tmp3, ctx);
1422                emit_branch(BNE, 4, 0, ctx);
1423                emit_nop(ctx);
1424                break;
1425        }
1426
1427        default:
1428                pr_err_once("unknown opcode %02x\n", code);
1429                return -EINVAL;
1430        }
1431
1432        return 0;
1433}
1434
1435static int build_body(struct jit_ctx *ctx)
1436{
1437        const struct bpf_prog *prog = ctx->prog;
1438        int i;
1439
1440        for (i = 0; i < prog->len; i++) {
1441                const struct bpf_insn *insn = &prog->insnsi[i];
1442                int ret;
1443
1444                ret = build_insn(insn, ctx);
1445
1446                if (ret > 0) {
1447                        i++;
1448                        ctx->offset[i] = ctx->idx;
1449                        continue;
1450                }
1451                ctx->offset[i] = ctx->idx;
1452                if (ret)
1453                        return ret;
1454        }
1455        return 0;
1456}
1457
1458static void jit_fill_hole(void *area, unsigned int size)
1459{
1460        u32 *ptr;
1461        /* We are guaranteed to have aligned memory. */
1462        for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1463                *ptr++ = 0x91d02005; /* ta 5 */
1464}
1465
1466bool bpf_jit_needs_zext(void)
1467{
1468        return true;
1469}
1470
1471struct sparc64_jit_data {
1472        struct bpf_binary_header *header;
1473        u8 *image;
1474        struct jit_ctx ctx;
1475};
1476
1477struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1478{
1479        struct bpf_prog *tmp, *orig_prog = prog;
1480        struct sparc64_jit_data *jit_data;
1481        struct bpf_binary_header *header;
1482        u32 prev_image_size, image_size;
1483        bool tmp_blinded = false;
1484        bool extra_pass = false;
1485        struct jit_ctx ctx;
1486        u8 *image_ptr;
1487        int pass, i;
1488
1489        if (!prog->jit_requested)
1490                return orig_prog;
1491
1492        tmp = bpf_jit_blind_constants(prog);
1493        /* If blinding was requested and we failed during blinding,
1494         * we must fall back to the interpreter.
1495         */
1496        if (IS_ERR(tmp))
1497                return orig_prog;
1498        if (tmp != prog) {
1499                tmp_blinded = true;
1500                prog = tmp;
1501        }
1502
1503        jit_data = prog->aux->jit_data;
1504        if (!jit_data) {
1505                jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1506                if (!jit_data) {
1507                        prog = orig_prog;
1508                        goto out;
1509                }
1510                prog->aux->jit_data = jit_data;
1511        }
1512        if (jit_data->ctx.offset) {
1513                ctx = jit_data->ctx;
1514                image_ptr = jit_data->image;
1515                header = jit_data->header;
1516                extra_pass = true;
1517                image_size = sizeof(u32) * ctx.idx;
1518                prev_image_size = image_size;
1519                pass = 1;
1520                goto skip_init_ctx;
1521        }
1522
1523        memset(&ctx, 0, sizeof(ctx));
1524        ctx.prog = prog;
1525
1526        ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL);
1527        if (ctx.offset == NULL) {
1528                prog = orig_prog;
1529                goto out_off;
1530        }
1531
1532        /* Longest sequence emitted is for bswap32, 12 instructions.  Pre-cook
1533         * the offset array so that we converge faster.
1534         */
1535        for (i = 0; i < prog->len; i++)
1536                ctx.offset[i] = i * (12 * 4);
1537
1538        prev_image_size = ~0U;
1539        for (pass = 1; pass < 40; pass++) {
1540                ctx.idx = 0;
1541
1542                build_prologue(&ctx);
1543                if (build_body(&ctx)) {
1544                        prog = orig_prog;
1545                        goto out_off;
1546                }
1547                build_epilogue(&ctx);
1548
1549                if (bpf_jit_enable > 1)
1550                        pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass,
1551                                ctx.idx * 4,
1552                                ctx.tmp_1_used ? '1' : ' ',
1553                                ctx.tmp_2_used ? '2' : ' ',
1554                                ctx.tmp_3_used ? '3' : ' ',
1555                                ctx.saw_frame_pointer ? 'F' : ' ',
1556                                ctx.saw_call ? 'C' : ' ',
1557                                ctx.saw_tail_call ? 'T' : ' ');
1558
1559                if (ctx.idx * 4 == prev_image_size)
1560                        break;
1561                prev_image_size = ctx.idx * 4;
1562                cond_resched();
1563        }
1564
1565        /* Now we know the actual image size. */
1566        image_size = sizeof(u32) * ctx.idx;
1567        header = bpf_jit_binary_alloc(image_size, &image_ptr,
1568                                      sizeof(u32), jit_fill_hole);
1569        if (header == NULL) {
1570                prog = orig_prog;
1571                goto out_off;
1572        }
1573
1574        ctx.image = (u32 *)image_ptr;
1575skip_init_ctx:
1576        ctx.idx = 0;
1577
1578        build_prologue(&ctx);
1579
1580        if (build_body(&ctx)) {
1581                bpf_jit_binary_free(header);
1582                prog = orig_prog;
1583                goto out_off;
1584        }
1585
1586        build_epilogue(&ctx);
1587
1588        if (ctx.idx * 4 != prev_image_size) {
1589                pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n",
1590                       prev_image_size, ctx.idx * 4);
1591                bpf_jit_binary_free(header);
1592                prog = orig_prog;
1593                goto out_off;
1594        }
1595
1596        if (bpf_jit_enable > 1)
1597                bpf_jit_dump(prog->len, image_size, pass, ctx.image);
1598
1599        bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE));
1600
1601        if (!prog->is_func || extra_pass) {
1602                bpf_jit_binary_lock_ro(header);
1603        } else {
1604                jit_data->ctx = ctx;
1605                jit_data->image = image_ptr;
1606                jit_data->header = header;
1607        }
1608
1609        prog->bpf_func = (void *)ctx.image;
1610        prog->jited = 1;
1611        prog->jited_len = image_size;
1612
1613        if (!prog->is_func || extra_pass) {
1614                bpf_prog_fill_jited_linfo(prog, ctx.offset);
1615out_off:
1616                kfree(ctx.offset);
1617                kfree(jit_data);
1618                prog->aux->jit_data = NULL;
1619        }
1620out:
1621        if (tmp_blinded)
1622                bpf_jit_prog_release_other(prog, prog == orig_prog ?
1623                                           tmp : orig_prog);
1624        return prog;
1625}
1626