LXR linux/arch/s390/net/bpf_jit

   1/*
   2 * BPF Jit compiler for s390.
   3 *
   4 * Copyright IBM Corp. 2012
   5 *
   6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
   7 */
   8#include <linux/moduleloader.h>
   9#include <linux/netdevice.h>
  10#include <linux/if_vlan.h>
  11#include <linux/filter.h>
  12#include <asm/cacheflush.h>
  13#include <asm/processor.h>
  14#include <asm/facility.h>
  15
  16/*
  17 * Conventions:
  18 *   %r2 = skb pointer
  19 *   %r3 = offset parameter
  20 *   %r4 = scratch register / length parameter
  21 *   %r5 = BPF A accumulator
  22 *   %r8 = return address
  23 *   %r9 = save register for skb pointer
  24 *   %r10 = skb->data
  25 *   %r11 = skb->len - skb->data_len (headlen)
  26 *   %r12 = BPF X accumulator
  27 *   %r13 = literal pool pointer
  28 *   0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS
  29 */
  30int bpf_jit_enable __read_mostly;
  31
  32/*
  33 * assembly code in arch/x86/net/bpf_jit.S
  34 */
  35extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[];
  36extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[];
  37
  38struct bpf_jit {
  39        unsigned int seen;
  40        u8 *start;
  41        u8 *prg;
  42        u8 *mid;
  43        u8 *lit;
  44        u8 *end;
  45        u8 *base_ip;
  46        u8 *ret0_ip;
  47        u8 *exit_ip;
  48        unsigned int off_load_word;
  49        unsigned int off_load_half;
  50        unsigned int off_load_byte;
  51        unsigned int off_load_bmsh;
  52        unsigned int off_load_iword;
  53        unsigned int off_load_ihalf;
  54        unsigned int off_load_ibyte;
  55};
  56
  57#define BPF_SIZE_MAX    4096    /* Max size for program */
  58
  59#define SEEN_DATAREF    1       /* might call external helpers */
  60#define SEEN_XREG       2       /* ebx is used */
  61#define SEEN_MEM        4       /* use mem[] for temporary storage */
  62#define SEEN_RET0       8       /* pc_ret0 points to a valid return 0 */
  63#define SEEN_LITERAL    16      /* code uses literals */
  64#define SEEN_LOAD_WORD  32      /* code uses sk_load_word */
  65#define SEEN_LOAD_HALF  64      /* code uses sk_load_half */
  66#define SEEN_LOAD_BYTE  128     /* code uses sk_load_byte */
  67#define SEEN_LOAD_BMSH  256     /* code uses sk_load_byte_msh */
  68#define SEEN_LOAD_IWORD 512     /* code uses sk_load_word_ind */
  69#define SEEN_LOAD_IHALF 1024    /* code uses sk_load_half_ind */
  70#define SEEN_LOAD_IBYTE 2048    /* code uses sk_load_byte_ind */
  71
  72#define EMIT2(op)                                       \
  73({                                                      \
  74        if (jit->prg + 2 <= jit->mid)                   \
  75                *(u16 *) jit->prg = op;                 \
  76        jit->prg += 2;                                  \
  77})
  78
  79#define EMIT4(op)                                       \
  80({                                                      \
  81        if (jit->prg + 4 <= jit->mid)                   \
  82                *(u32 *) jit->prg = op;                 \
  83        jit->prg += 4;                                  \
  84})
  85
  86#define EMIT4_DISP(op, disp)                            \
  87({                                                      \
  88        unsigned int __disp = (disp) & 0xfff;           \
  89        EMIT4(op | __disp);                             \
  90})
  91
  92#define EMIT4_IMM(op, imm)                              \
  93({                                                      \
  94        unsigned int __imm = (imm) & 0xffff;            \
  95        EMIT4(op | __imm);                              \
  96})
  97
  98#define EMIT4_PCREL(op, pcrel)                          \
  99({                                                      \
 100        long __pcrel = ((pcrel) >> 1) & 0xffff;         \
 101        EMIT4(op | __pcrel);                            \
 102})
 103
 104#define EMIT6(op1, op2)                                 \
 105({                                                      \
 106        if (jit->prg + 6 <= jit->mid) {                 \
 107                *(u32 *) jit->prg = op1;                \
 108                *(u16 *) (jit->prg + 4) = op2;          \
 109        }                                               \
 110        jit->prg += 6;                                  \
 111})
 112
 113#define EMIT6_DISP(op1, op2, disp)                      \
 114({                                                      \
 115        unsigned int __disp = (disp) & 0xfff;           \
 116        EMIT6(op1 | __disp, op2);                       \
 117})
 118
 119#define EMIT6_IMM(op, imm)                              \
 120({                                                      \
 121        unsigned int __imm = (imm);                     \
 122        EMIT6(op | (__imm >> 16), __imm & 0xffff);      \
 123})
 124
 125#define EMIT_CONST(val)                                 \
 126({                                                      \
 127        unsigned int ret;                               \
 128        ret = (unsigned int) (jit->lit - jit->base_ip); \
 129        jit->seen |= SEEN_LITERAL;                      \
 130        if (jit->lit + 4 <= jit->end)                   \
 131                *(u32 *) jit->lit = val;                \
 132        jit->lit += 4;                                  \
 133        ret;                                            \
 134})
 135
 136#define EMIT_FN_CONST(bit, fn)                          \
 137({                                                      \
 138        unsigned int ret;                               \
 139        ret = (unsigned int) (jit->lit - jit->base_ip); \
 140        if (jit->seen & bit) {                          \
 141                jit->seen |= SEEN_LITERAL;              \
 142                if (jit->lit + 8 <= jit->end)           \
 143                        *(void **) jit->lit = fn;       \
 144                jit->lit += 8;                          \
 145        }                                               \
 146        ret;                                            \
 147})
 148
 149static void bpf_jit_prologue(struct bpf_jit *jit)
 150{
 151        /* Save registers and create stack frame if necessary */
 152        if (jit->seen & SEEN_DATAREF) {
 153                /* stmg %r8,%r15,88(%r15) */
 154                EMIT6(0xeb8ff058, 0x0024);
 155                /* lgr %r14,%r15 */
 156                EMIT4(0xb90400ef);
 157                /* ahi %r15,<offset> */
 158                EMIT4_IMM(0xa7fa0000, (jit->seen & SEEN_MEM) ? -112 : -80);
 159                /* stg %r14,152(%r15) */
 160                EMIT6(0xe3e0f098, 0x0024);
 161        } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL))
 162                /* stmg %r12,%r13,120(%r15) */
 163                EMIT6(0xebcdf078, 0x0024);
 164        else if (jit->seen & SEEN_XREG)
 165                /* stg %r12,120(%r15) */
 166                EMIT6(0xe3c0f078, 0x0024);
 167        else if (jit->seen & SEEN_LITERAL)
 168                /* stg %r13,128(%r15) */
 169                EMIT6(0xe3d0f080, 0x0024);
 170
 171        /* Setup literal pool */
 172        if (jit->seen & SEEN_LITERAL) {
 173                /* basr %r13,0 */
 174                EMIT2(0x0dd0);
 175                jit->base_ip = jit->prg;
 176        }
 177        jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word);
 178        jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half);
 179        jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte);
 180        jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh);
 181        jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind);
 182        jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind);
 183        jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind);
 184
 185        /* Filter needs to access skb data */
 186        if (jit->seen & SEEN_DATAREF) {
 187                /* l %r11,<len>(%r2) */
 188                EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len));
 189                /* s %r11,<data_len>(%r2) */
 190                EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len));
 191                /* lg %r10,<data>(%r2) */
 192                EMIT6_DISP(0xe3a02000, 0x0004,
 193                           offsetof(struct sk_buff, data));
 194        }
 195}
 196
 197static void bpf_jit_epilogue(struct bpf_jit *jit)
 198{
 199        /* Return 0 */
 200        if (jit->seen & SEEN_RET0) {
 201                jit->ret0_ip = jit->prg;
 202                /* lghi %r2,0 */
 203                EMIT4(0xa7290000);
 204        }
 205        jit->exit_ip = jit->prg;
 206        /* Restore registers */
 207        if (jit->seen & SEEN_DATAREF)
 208                /* lmg %r8,%r15,<offset>(%r15) */
 209                EMIT6_DISP(0xeb8ff000, 0x0004,
 210                           (jit->seen & SEEN_MEM) ? 200 : 168);
 211        else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL))
 212                /* lmg %r12,%r13,120(%r15) */
 213                EMIT6(0xebcdf078, 0x0004);
 214        else if (jit->seen & SEEN_XREG)
 215                /* lg %r12,120(%r15) */
 216                EMIT6(0xe3c0f078, 0x0004);
 217        else if (jit->seen & SEEN_LITERAL)
 218                /* lg %r13,128(%r15) */
 219                EMIT6(0xe3d0f080, 0x0004);
 220        /* br %r14 */
 221        EMIT2(0x07fe);
 222}
 223
 224/*
 225 * make sure we dont leak kernel information to user
 226 */
 227static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter)
 228{
 229        /* Clear temporary memory if (seen & SEEN_MEM) */
 230        if (jit->seen & SEEN_MEM)
 231                /* xc 0(64,%r15),0(%r15) */
 232                EMIT6(0xd73ff000, 0xf000);
 233        /* Clear X if (seen & SEEN_XREG) */
 234        if (jit->seen & SEEN_XREG)
 235                /* lhi %r12,0 */
 236                EMIT4(0xa7c80000);
 237        /* Clear A if the first register does not set it. */
 238        switch (filter[0].code) {
 239        case BPF_S_LD_W_ABS:
 240        case BPF_S_LD_H_ABS:
 241        case BPF_S_LD_B_ABS:
 242        case BPF_S_LD_W_LEN:
 243        case BPF_S_LD_W_IND:
 244        case BPF_S_LD_H_IND:
 245        case BPF_S_LD_B_IND:
 246        case BPF_S_LDX_B_MSH:
 247        case BPF_S_LD_IMM:
 248        case BPF_S_LD_MEM:
 249        case BPF_S_MISC_TXA:
 250        case BPF_S_ANC_PROTOCOL:
 251        case BPF_S_ANC_PKTTYPE:
 252        case BPF_S_ANC_IFINDEX:
 253        case BPF_S_ANC_MARK:
 254        case BPF_S_ANC_QUEUE:
 255        case BPF_S_ANC_HATYPE:
 256        case BPF_S_ANC_RXHASH:
 257        case BPF_S_ANC_CPU:
 258        case BPF_S_ANC_VLAN_TAG:
 259        case BPF_S_ANC_VLAN_TAG_PRESENT:
 260        case BPF_S_RET_K:
 261                /* first instruction sets A register */
 262                break;
 263        default: /* A = 0 */
 264                /* lhi %r5,0 */
 265                EMIT4(0xa7580000);
 266        }
 267}
 268
 269static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter,
 270                        unsigned int *addrs, int i, int last)
 271{
 272        unsigned int K;
 273        int offset;
 274        unsigned int mask;
 275
 276        K = filter->k;
 277        switch (filter->code) {
 278        case BPF_S_ALU_ADD_X: /* A += X */
 279                jit->seen |= SEEN_XREG;
 280                /* ar %r5,%r12 */
 281                EMIT2(0x1a5c);
 282                break;
 283        case BPF_S_ALU_ADD_K: /* A += K */
 284                if (!K)
 285                        break;
 286                if (K <= 16383)
 287                        /* ahi %r5,<K> */
 288                        EMIT4_IMM(0xa75a0000, K);
 289                else if (test_facility(21))
 290                        /* alfi %r5,<K> */
 291                        EMIT6_IMM(0xc25b0000, K);
 292                else
 293                        /* a %r5,<d(K)>(%r13) */
 294                        EMIT4_DISP(0x5a50d000, EMIT_CONST(K));
 295                break;
 296        case BPF_S_ALU_SUB_X: /* A -= X */
 297                jit->seen |= SEEN_XREG;
 298                /* sr %r5,%r12 */
 299                EMIT2(0x1b5c);
 300                break;
 301        case BPF_S_ALU_SUB_K: /* A -= K */
 302                if (!K)
 303                        break;
 304                if (K <= 16384)
 305                        /* ahi %r5,-K */
 306                        EMIT4_IMM(0xa75a0000, -K);
 307                else if (test_facility(21))
 308                        /* alfi %r5,-K */
 309                        EMIT6_IMM(0xc25b0000, -K);
 310                else
 311                        /* s %r5,<d(K)>(%r13) */
 312                        EMIT4_DISP(0x5b50d000, EMIT_CONST(K));
 313                break;
 314        case BPF_S_ALU_MUL_X: /* A *= X */
 315                jit->seen |= SEEN_XREG;
 316                /* msr %r5,%r12 */
 317                EMIT4(0xb252005c);
 318                break;
 319        case BPF_S_ALU_MUL_K: /* A *= K */
 320                if (K <= 16383)
 321                        /* mhi %r5,K */
 322                        EMIT4_IMM(0xa75c0000, K);
 323                else if (test_facility(34))
 324                        /* msfi %r5,<K> */
 325                        EMIT6_IMM(0xc2510000, K);
 326                else
 327                        /* ms %r5,<d(K)>(%r13) */
 328                        EMIT4_DISP(0x7150d000, EMIT_CONST(K));
 329                break;
 330        case BPF_S_ALU_DIV_X: /* A /= X */
 331                jit->seen |= SEEN_XREG | SEEN_RET0;
 332                /* ltr %r12,%r12 */
 333                EMIT2(0x12cc);
 334                /* jz <ret0> */
 335                EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg));
 336                /* lhi %r4,0 */
 337                EMIT4(0xa7480000);
 338                /* dr %r4,%r12 */
 339                EMIT2(0x1d4c);
 340                break;
 341        case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K) */
 342                /* m %r4,<d(K)>(%r13) */
 343                EMIT4_DISP(0x5c40d000, EMIT_CONST(K));
 344                /* lr %r5,%r4 */
 345                EMIT2(0x1854);
 346                break;
 347        case BPF_S_ALU_MOD_X: /* A %= X */
 348                jit->seen |= SEEN_XREG | SEEN_RET0;
 349                /* ltr %r12,%r12 */
 350                EMIT2(0x12cc);
 351                /* jz <ret0> */
 352                EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg));
 353                /* lhi %r4,0 */
 354                EMIT4(0xa7480000);
 355                /* dr %r4,%r12 */
 356                EMIT2(0x1d4c);
 357                /* lr %r5,%r4 */
 358                EMIT2(0x1854);
 359                break;
 360        case BPF_S_ALU_MOD_K: /* A %= K */
 361                /* lhi %r4,0 */
 362                EMIT4(0xa7480000);
 363                /* d %r4,<d(K)>(%r13) */
 364                EMIT4_DISP(0x5d40d000, EMIT_CONST(K));
 365                /* lr %r5,%r4 */
 366                EMIT2(0x1854);
 367                break;
 368        case BPF_S_ALU_AND_X: /* A &= X */
 369                jit->seen |= SEEN_XREG;
 370                /* nr %r5,%r12 */
 371                EMIT2(0x145c);
 372                break;
 373        case BPF_S_ALU_AND_K: /* A &= K */
 374                if (test_facility(21))
 375                        /* nilf %r5,<K> */
 376                        EMIT6_IMM(0xc05b0000, K);
 377                else
 378                        /* n %r5,<d(K)>(%r13) */
 379                        EMIT4_DISP(0x5450d000, EMIT_CONST(K));
 380                break;
 381        case BPF_S_ALU_OR_X: /* A |= X */
 382                jit->seen |= SEEN_XREG;
 383                /* or %r5,%r12 */
 384                EMIT2(0x165c);
 385                break;
 386        case BPF_S_ALU_OR_K: /* A |= K */
 387                if (test_facility(21))
 388                        /* oilf %r5,<K> */
 389                        EMIT6_IMM(0xc05d0000, K);
 390                else
 391                        /* o %r5,<d(K)>(%r13) */
 392                        EMIT4_DISP(0x5650d000, EMIT_CONST(K));
 393                break;
 394        case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */
 395        case BPF_S_ALU_XOR_X:
 396                jit->seen |= SEEN_XREG;
 397                /* xr %r5,%r12 */
 398                EMIT2(0x175c);
 399                break;
 400        case BPF_S_ALU_XOR_K: /* A ^= K */
 401                if (!K)
 402                        break;
 403                /* x %r5,<d(K)>(%r13) */
 404                EMIT4_DISP(0x5750d000, EMIT_CONST(K));
 405                break;
 406        case BPF_S_ALU_LSH_X: /* A <<= X; */
 407                jit->seen |= SEEN_XREG;
 408                /* sll %r5,0(%r12) */
 409                EMIT4(0x8950c000);
 410                break;
 411        case BPF_S_ALU_LSH_K: /* A <<= K */
 412                if (K == 0)
 413                        break;
 414                /* sll %r5,K */
 415                EMIT4_DISP(0x89500000, K);
 416                break;
 417        case BPF_S_ALU_RSH_X: /* A >>= X; */
 418                jit->seen |= SEEN_XREG;
 419                /* srl %r5,0(%r12) */
 420                EMIT4(0x8850c000);
 421                break;
 422        case BPF_S_ALU_RSH_K: /* A >>= K; */
 423                if (K == 0)
 424                        break;
 425                /* srl %r5,K */
 426                EMIT4_DISP(0x88500000, K);
 427                break;
 428        case BPF_S_ALU_NEG: /* A = -A */
 429                /* lnr %r5,%r5 */
 430                EMIT2(0x1155);
 431                break;
 432        case BPF_S_JMP_JA: /* ip += K */
 433                offset = addrs[i + K] + jit->start - jit->prg;
 434                EMIT4_PCREL(0xa7f40000, offset);
 435                break;
 436        case BPF_S_JMP_JGT_K: /* ip += (A > K) ? jt : jf */
 437                mask = 0x200000; /* jh */
 438                goto kbranch;
 439        case BPF_S_JMP_JGE_K: /* ip += (A >= K) ? jt : jf */
 440                mask = 0xa00000; /* jhe */
 441                goto kbranch;
 442        case BPF_S_JMP_JEQ_K: /* ip += (A == K) ? jt : jf */
 443                mask = 0x800000; /* je */
 444kbranch:        /* Emit compare if the branch targets are different */
 445                if (filter->jt != filter->jf) {
 446                        if (K <= 16383)
 447                                /* chi %r5,<K> */
 448                                EMIT4_IMM(0xa75e0000, K);
 449                        else if (test_facility(21))
 450                                /* clfi %r5,<K> */
 451                                EMIT6_IMM(0xc25f0000, K);
 452                        else
 453                                /* c %r5,<d(K)>(%r13) */
 454                                EMIT4_DISP(0x5950d000, EMIT_CONST(K));
 455                }
 456branch:         if (filter->jt == filter->jf) {
 457                        if (filter->jt == 0)
 458                                break;
 459                        /* j <jt> */
 460                        offset = addrs[i + filter->jt] + jit->start - jit->prg;
 461                        EMIT4_PCREL(0xa7f40000, offset);
 462                        break;
 463                }
 464                if (filter->jt != 0) {
 465                        /* brc  <mask>,<jt> */
 466                        offset = addrs[i + filter->jt] + jit->start - jit->prg;
 467                        EMIT4_PCREL(0xa7040000 | mask, offset);
 468                }
 469                if (filter->jf != 0) {
 470                        /* brc  <mask^15>,<jf> */
 471                        offset = addrs[i + filter->jf] + jit->start - jit->prg;
 472                        EMIT4_PCREL(0xa7040000 | (mask ^ 0xf00000), offset);
 473                }
 474                break;
 475        case BPF_S_JMP_JSET_K: /* ip += (A & K) ? jt : jf */
 476                mask = 0x700000; /* jnz */
 477                /* Emit test if the branch targets are different */
 478                if (filter->jt != filter->jf) {
 479                        if (K > 65535) {
 480                                /* lr %r4,%r5 */
 481                                EMIT2(0x1845);
 482                                /* n %r4,<d(K)>(%r13) */
 483                                EMIT4_DISP(0x5440d000, EMIT_CONST(K));
 484                        } else
 485                                /* tmll %r5,K */
 486                                EMIT4_IMM(0xa7510000, K);
 487                }
 488                goto branch;
 489        case BPF_S_JMP_JGT_X: /* ip += (A > X) ? jt : jf */
 490                mask = 0x200000; /* jh */
 491                goto xbranch;
 492        case BPF_S_JMP_JGE_X: /* ip += (A >= X) ? jt : jf */
 493                mask = 0xa00000; /* jhe */
 494                goto xbranch;
 495        case BPF_S_JMP_JEQ_X: /* ip += (A == X) ? jt : jf */
 496                mask = 0x800000; /* je */
 497xbranch:        /* Emit compare if the branch targets are different */
 498                if (filter->jt != filter->jf) {
 499                        jit->seen |= SEEN_XREG;
 500                        /* cr %r5,%r12 */
 501                        EMIT2(0x195c);
 502                }
 503                goto branch;
 504        case BPF_S_JMP_JSET_X: /* ip += (A & X) ? jt : jf */
 505                mask = 0x700000; /* jnz */
 506                /* Emit test if the branch targets are different */
 507                if (filter->jt != filter->jf) {
 508                        jit->seen |= SEEN_XREG;
 509                        /* lr %r4,%r5 */
 510                        EMIT2(0x1845);
 511                        /* nr %r4,%r12 */
 512                        EMIT2(0x144c);
 513                }
 514                goto branch;
 515        case BPF_S_LD_W_ABS: /* A = *(u32 *) (skb->data+K) */
 516                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_WORD;
 517                offset = jit->off_load_word;
 518                goto load_abs;
 519        case BPF_S_LD_H_ABS: /* A = *(u16 *) (skb->data+K) */
 520                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_HALF;
 521                offset = jit->off_load_half;
 522                goto load_abs;
 523        case BPF_S_LD_B_ABS: /* A = *(u8 *) (skb->data+K) */
 524                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_BYTE;
 525                offset = jit->off_load_byte;
 526load_abs:       if ((int) K < 0)
 527                        goto out;
 528call_fn:        /* lg %r1,<d(function)>(%r13) */
 529                EMIT6_DISP(0xe310d000, 0x0004, offset);
 530                /* l %r3,<d(K)>(%r13) */
 531                EMIT4_DISP(0x5830d000, EMIT_CONST(K));
 532                /* basr %r8,%r1 */
 533                EMIT2(0x0d81);
 534                /* jnz <ret0> */
 535                EMIT4_PCREL(0xa7740000, (jit->ret0_ip - jit->prg));
 536                break;
 537        case BPF_S_LD_W_IND: /* A = *(u32 *) (skb->data+K+X) */
 538                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IWORD;
 539                offset = jit->off_load_iword;
 540                goto call_fn;
 541        case BPF_S_LD_H_IND: /* A = *(u16 *) (skb->data+K+X) */
 542                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IHALF;
 543                offset = jit->off_load_ihalf;
 544                goto call_fn;
 545        case BPF_S_LD_B_IND: /* A = *(u8 *) (skb->data+K+X) */
 546                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IBYTE;
 547                offset = jit->off_load_ibyte;
 548                goto call_fn;
 549        case BPF_S_LDX_B_MSH:
 550                /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */
 551                jit->seen |= SEEN_RET0;
 552                if ((int) K < 0) {
 553                        /* j <ret0> */
 554                        EMIT4_PCREL(0xa7f40000, (jit->ret0_ip - jit->prg));
 555                        break;
 556                }
 557                jit->seen |= SEEN_DATAREF | SEEN_LOAD_BMSH;
 558                offset = jit->off_load_bmsh;
 559                goto call_fn;
 560        case BPF_S_LD_W_LEN: /* A = skb->len; */
 561                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
 562                /* l %r5,<d(len)>(%r2) */
 563                EMIT4_DISP(0x58502000, offsetof(struct sk_buff, len));
 564                break;
 565        case BPF_S_LDX_W_LEN: /* X = skb->len; */
 566                jit->seen |= SEEN_XREG;
 567                /* l %r12,<d(len)>(%r2) */
 568                EMIT4_DISP(0x58c02000, offsetof(struct sk_buff, len));
 569                break;
 570        case BPF_S_LD_IMM: /* A = K */
 571                if (K <= 16383)
 572                        /* lhi %r5,K */
 573                        EMIT4_IMM(0xa7580000, K);
 574                else if (test_facility(21))
 575                        /* llilf %r5,<K> */
 576                        EMIT6_IMM(0xc05f0000, K);
 577                else
 578                        /* l %r5,<d(K)>(%r13) */
 579                        EMIT4_DISP(0x5850d000, EMIT_CONST(K));
 580                break;
 581        case BPF_S_LDX_IMM: /* X = K */
 582                jit->seen |= SEEN_XREG;
 583                if (K <= 16383)
 584                        /* lhi %r12,<K> */
 585                        EMIT4_IMM(0xa7c80000, K);
 586                else if (test_facility(21))
 587                        /* llilf %r12,<K> */
 588                        EMIT6_IMM(0xc0cf0000, K);
 589                else
 590                        /* l %r12,<d(K)>(%r13) */
 591                        EMIT4_DISP(0x58c0d000, EMIT_CONST(K));
 592                break;
 593        case BPF_S_LD_MEM: /* A = mem[K] */
 594                jit->seen |= SEEN_MEM;
 595                /* l %r5,<K>(%r15) */
 596                EMIT4_DISP(0x5850f000,
 597                           (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4);
 598                break;
 599        case BPF_S_LDX_MEM: /* X = mem[K] */
 600                jit->seen |= SEEN_XREG | SEEN_MEM;
 601                /* l %r12,<K>(%r15) */
 602                EMIT4_DISP(0x58c0f000,
 603                           (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4);
 604                break;
 605        case BPF_S_MISC_TAX: /* X = A */
 606                jit->seen |= SEEN_XREG;
 607                /* lr %r12,%r5 */
 608                EMIT2(0x18c5);
 609                break;
 610        case BPF_S_MISC_TXA: /* A = X */
 611                jit->seen |= SEEN_XREG;
 612                /* lr %r5,%r12 */
 613                EMIT2(0x185c);
 614                break;
 615        case BPF_S_RET_K:
 616                if (K == 0) {
 617                        jit->seen |= SEEN_RET0;
 618                        if (last)
 619                                break;
 620                        /* j <ret0> */
 621                        EMIT4_PCREL(0xa7f40000, jit->ret0_ip - jit->prg);
 622                } else {
 623                        if (K <= 16383)
 624                                /* lghi %r2,K */
 625                                EMIT4_IMM(0xa7290000, K);
 626                        else
 627                                /* llgf %r2,<K>(%r13) */
 628                                EMIT6_DISP(0xe320d000, 0x0016, EMIT_CONST(K));
 629                        /* j <exit> */
 630                        if (last && !(jit->seen & SEEN_RET0))
 631                                break;
 632                        EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);
 633                }
 634                break;
 635        case BPF_S_RET_A:
 636                /* llgfr %r2,%r5 */
 637                EMIT4(0xb9160025);
 638                /* j <exit> */
 639                EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);
 640                break;
 641        case BPF_S_ST: /* mem[K] = A */
 642                jit->seen |= SEEN_MEM;
 643                /* st %r5,<K>(%r15) */
 644                EMIT4_DISP(0x5050f000,
 645                           (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4);
 646                break;
 647        case BPF_S_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */
 648                jit->seen |= SEEN_XREG | SEEN_MEM;
 649                /* st %r12,<K>(%r15) */
 650                EMIT4_DISP(0x50c0f000,
 651                           (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4);
 652                break;
 653        case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */
 654                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
 655                /* lhi %r5,0 */
 656                EMIT4(0xa7580000);
 657                /* icm  %r5,3,<d(protocol)>(%r2) */
 658                EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol));
 659                break;
 660        case BPF_S_ANC_IFINDEX: /* if (!skb->dev) return 0;
 661                                 * A = skb->dev->ifindex */
 662                BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
 663                jit->seen |= SEEN_RET0;
 664                /* lg %r1,<d(dev)>(%r2) */
 665                EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev));
 666                /* ltgr %r1,%r1 */
 667                EMIT4(0xb9020011);
 668                /* jz <ret0> */
 669                EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
 670                /* l %r5,<d(ifindex)>(%r1) */
 671                EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex));
 672                break;
 673        case BPF_S_ANC_MARK: /* A = skb->mark */
 674                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
 675                /* l %r5,<d(mark)>(%r2) */
 676                EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark));
 677                break;
 678        case BPF_S_ANC_QUEUE: /* A = skb->queue_mapping */
 679                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
 680                /* lhi %r5,0 */
 681                EMIT4(0xa7580000);
 682                /* icm  %r5,3,<d(queue_mapping)>(%r2) */
 683                EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping));
 684                break;
 685        case BPF_S_ANC_HATYPE:  /* if (!skb->dev) return 0;
 686                                 * A = skb->dev->type */
 687                BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2);
 688                jit->seen |= SEEN_RET0;
 689                /* lg %r1,<d(dev)>(%r2) */
 690                EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev));
 691                /* ltgr %r1,%r1 */
 692                EMIT4(0xb9020011);
 693                /* jz <ret0> */
 694                EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
 695                /* lhi %r5,0 */
 696                EMIT4(0xa7580000);
 697                /* icm  %r5,3,<d(type)>(%r1) */
 698                EMIT4_DISP(0xbf531000, offsetof(struct net_device, type));
 699                break;
 700        case BPF_S_ANC_RXHASH: /* A = skb->rxhash */
 701                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4);
 702                /* l %r5,<d(rxhash)>(%r2) */
 703                EMIT4_DISP(0x58502000, offsetof(struct sk_buff, rxhash));
 704                break;
 705        case BPF_S_ANC_VLAN_TAG:
 706        case BPF_S_ANC_VLAN_TAG_PRESENT:
 707                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
 708                BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
 709                /* lhi %r5,0 */
 710                EMIT4(0xa7580000);
 711                /* icm  %r5,3,<d(vlan_tci)>(%r2) */
 712                EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, vlan_tci));
 713                if (filter->code == BPF_S_ANC_VLAN_TAG) {
 714                        /* nill %r5,0xefff */
 715                        EMIT4_IMM(0xa5570000, ~VLAN_TAG_PRESENT);
 716                } else {
 717                        /* nill %r5,0x1000 */
 718                        EMIT4_IMM(0xa5570000, VLAN_TAG_PRESENT);
 719                        /* srl %r5,12 */
 720                        EMIT4_DISP(0x88500000, 12);
 721                }
 722                break;
 723        case BPF_S_ANC_CPU: /* A = smp_processor_id() */
 724#ifdef CONFIG_SMP
 725                /* l %r5,<d(cpu_nr)> */
 726                EMIT4_DISP(0x58500000, offsetof(struct _lowcore, cpu_nr));
 727#else
 728                /* lhi %r5,0 */
 729                EMIT4(0xa7580000);
 730#endif
 731                break;
 732        default: /* too complex, give up */
 733                goto out;
 734        }
 735        addrs[i] = jit->prg - jit->start;
 736        return 0;
 737out:
 738        return -1;
 739}
 740
 741void bpf_jit_compile(struct sk_filter *fp)
 742{
 743        unsigned long size, prg_len, lit_len;
 744        struct bpf_jit jit, cjit;
 745        unsigned int *addrs;
 746        int pass, i;
 747
 748        if (!bpf_jit_enable)
 749                return;
 750        addrs = kmalloc(fp->len * sizeof(*addrs), GFP_KERNEL);
 751        if (addrs == NULL)
 752                return;
 753        memset(addrs, 0, fp->len * sizeof(*addrs));
 754        memset(&jit, 0, sizeof(cjit));
 755        memset(&cjit, 0, sizeof(cjit));
 756
 757        for (pass = 0; pass < 10; pass++) {
 758                jit.prg = jit.start;
 759                jit.lit = jit.mid;
 760
 761                bpf_jit_prologue(&jit);
 762                bpf_jit_noleaks(&jit, fp->insns);
 763                for (i = 0; i < fp->len; i++) {
 764                        if (bpf_jit_insn(&jit, fp->insns + i, addrs, i,
 765                                         i == fp->len - 1))
 766                                goto out;
 767                }
 768                bpf_jit_epilogue(&jit);
 769                if (jit.start) {
 770                        WARN_ON(jit.prg > cjit.prg || jit.lit > cjit.lit);
 771                        if (memcmp(&jit, &cjit, sizeof(jit)) == 0)
 772                                break;
 773                } else if (jit.prg == cjit.prg && jit.lit == cjit.lit) {
 774                        prg_len = jit.prg - jit.start;
 775                        lit_len = jit.lit - jit.mid;
 776                        size = max_t(unsigned long, prg_len + lit_len,
 777                                     sizeof(struct work_struct));
 778                        if (size >= BPF_SIZE_MAX)
 779                                goto out;
 780                        jit.start = module_alloc(size);
 781                        if (!jit.start)
 782                                goto out;
 783                        jit.prg = jit.mid = jit.start + prg_len;
 784                        jit.lit = jit.end = jit.start + prg_len + lit_len;
 785                        jit.base_ip += (unsigned long) jit.start;
 786                        jit.exit_ip += (unsigned long) jit.start;
 787                        jit.ret0_ip += (unsigned long) jit.start;
 788                }
 789                cjit = jit;
 790        }
 791        if (bpf_jit_enable > 1) {
 792                pr_err("flen=%d proglen=%lu pass=%d image=%p\n",
 793                       fp->len, jit.end - jit.start, pass, jit.start);
 794                if (jit.start) {
 795                        printk(KERN_ERR "JIT code:\n");
 796                        print_fn_code(jit.start, jit.mid - jit.start);
 797                        print_hex_dump(KERN_ERR, "JIT literals:\n",
 798                                       DUMP_PREFIX_ADDRESS, 16, 1,
 799                                       jit.mid, jit.end - jit.mid, false);
 800                }
 801        }
 802        if (jit.start)
 803                fp->bpf_func = (void *) jit.start;
 804out:
 805        kfree(addrs);
 806}
 807
 808static void jit_free_defer(struct work_struct *arg)
 809{
 810        module_free(NULL, arg);
 811}
 812
 813/* run from softirq, we must use a work_struct to call
 814 * module_free() from process context
 815 */
 816void bpf_jit_free(struct sk_filter *fp)
 817{
 818        struct work_struct *work;
 819
 820        if (fp->bpf_func == sk_run_filter)
 821                return;
 822        work = (struct work_struct *)fp->bpf_func;
 823        INIT_WORK(work, jit_free_defer);
 824        schedule_work(work);
 825}
 826